diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a41987ff..cd434a2a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,61 +1,54 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: trailing-whitespace - id: flake8 - id: check-json - id: check-yaml - repo: https://github.com/codespell-project/codespell rev: v1.16.0 hooks: - id: codespell exclude: > (?x)^( cypress/integration/directory.spec.js| swh/web/tests/resources/contents/code/extensions/.*| yarn.lock| package.json )$ - repo: local hooks: - id: mypy name: mypy entry: env DJANGO_SETTINGS_MODULE=swh.web.settings.development mypy args: [swh] pass_filenames: false language: system types: [python] - repo: local hooks: - id: eslint name: eslint entry: node_modules/.bin/eslint -c swh/web/assets/config/.eslintrc language: system types: [javascript] +- repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black + # unfortunately, we are far from being able to enable this... # - repo: https://github.com/PyCQA/pydocstyle.git # rev: 4.0.0 # hooks: # - id: pydocstyle # name: pydocstyle # description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. # entry: pydocstyle --convention=google # language: python # types: [python] - -# black requires py3.6+ -#- repo: https://github.com/python/black -# rev: 19.3b0 -# hooks: -# - id: black -# language_version: python3 -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.0.0-1 -# hooks: -# - id: blacken-docs -# additional_dependencies: [black==19.3b0] diff --git a/docs/conf.py b/docs/conf.py index 3149243b..933b2f9d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,15 +1,16 @@ # flake8: noqa import django import os -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'swh.web.settings.development') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.development") django.setup() import swh.docs.sphinx.conf as sphinx_conf from swh.web.doc_config import customize_sphinx_conf + customize_sphinx_conf(sphinx_conf) from swh.docs.sphinx.conf import * diff --git a/setup.cfg b/setup.cfg index 85bfba7a..20437a0a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,10 @@ [radon] exclude = swh/web/tests/resources/* + +[flake8] +# E203: whitespaces before ':' +# E231: missing whitespace after ',' +# W503: line break before binary operator +ignore = E203,E231,W503 +max-line-length = 88 + diff --git a/setup.py b/setup.py index 2c293178..9f0728c9 100755 --- a/setup.py +++ b/setup.py @@ -1,75 +1,75 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License v3 or later # See top-level LICENSE file for more information import os from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: +with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: - reqf = 'requirements-%s.txt' % name + reqf = "requirements-%s.txt" % name else: - reqf = 'requirements.txt' + reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue requirements.append(line) return requirements # package generated static assets as module data files data_files = [] -for root, _, files in os.walk('static/'): +for root, _, files in os.walk("static/"): root_files = [os.path.join(root, i) for i in files] - data_files.append((os.path.join('share/swh/web', root), root_files)) + data_files.append((os.path.join("share/swh/web", root), root_files)) setup( - name='swh.web', - description='Software Heritage Web UI', + name="swh.web", + description="Software Heritage Web UI", long_description=long_description, - long_description_content_type='text/markdown', - author='Software Heritage developers', - author_email='swh-devel@inria.fr', - url='https://forge.softwareheritage.org/diffusion/DWUI/', + long_description_content_type="text/markdown", + author="Software Heritage developers", + author_email="swh-devel@inria.fr", + url="https://forge.softwareheritage.org/diffusion/DWUI/", packages=find_packages(), scripts=[], - install_requires=parse_requirements() + parse_requirements('swh'), - setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + install_requires=parse_requirements() + parse_requirements("swh"), + setup_requires=["vcversioner"], + extras_require={"testing": parse_requirements("test")}, vcversioner={}, include_package_data=True, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", # noqa "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", "Framework :: Django", ], project_urls={ - 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', - 'Funding': 'https://www.softwareheritage.org/donate', - 'Source': 'https://forge.softwareheritage.org/source/swh-web', + "Bug Reports": "https://forge.softwareheritage.org/maniphest", + "Funding": "https://www.softwareheritage.org/donate", + "Source": "https://forge.softwareheritage.org/source/swh-web", }, - data_files=data_files + data_files=data_files, ) diff --git a/swh/web/admin/adminurls.py b/swh/web/admin/adminurls.py index 752cdc3e..3186114e 100644 --- a/swh/web/admin/adminurls.py +++ b/swh/web/admin/adminurls.py @@ -1,35 +1,35 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.urlsindex import UrlsIndex class AdminUrls(UrlsIndex): """ Class to manage swh-web admin urls. """ - scope = 'admin' + scope = "admin" def admin_route(*url_patterns, view_name=None): """ Decorator to ease the registration of a swh-web admin endpoint Args: url_patterns: list of url patterns used by Django to identify the admin routes view_name: the name of the Django view associated to the routes used to reverse the url """ - url_patterns = ['^' + url_pattern + '$' for url_pattern in url_patterns] + url_patterns = ["^" + url_pattern + "$" for url_pattern in url_patterns] def decorator(f): # register the route and its view in the browse endpoints index for url_pattern in url_patterns: AdminUrls.add_url_pattern(url_pattern, f, view_name) return f return decorator diff --git a/swh/web/admin/deposit.py b/swh/web/admin/deposit.py index d7da7bb5..bd02b595 100644 --- a/swh/web/admin/deposit.py +++ b/swh/web/admin/deposit.py @@ -1,95 +1,103 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import requests from django.core.cache import cache from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render from requests.auth import HTTPBasicAuth import sentry_sdk from swh.web.admin.adminurls import admin_route from swh.web.config import get_config -config = get_config()['deposit'] +config = get_config()["deposit"] -@admin_route(r'deposit/', view_name='admin-deposit') +@admin_route(r"deposit/", view_name="admin-deposit") @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save(request): - return render(request, 'admin/deposit.html') + return render(request, "admin/deposit.html") -@admin_route(r'deposit/list/', - view_name='admin-deposit-list') +@admin_route(r"deposit/list/", view_name="admin-deposit-list") @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_deposit_list(request): table_data = {} - table_data['draw'] = int(request.GET['draw']) - deposits_list_url = config['private_api_url'] + 'deposits' - deposits_list_auth = HTTPBasicAuth(config['private_api_user'], - config['private_api_password']) + table_data["draw"] = int(request.GET["draw"]) + deposits_list_url = config["private_api_url"] + "deposits" + deposits_list_auth = HTTPBasicAuth( + config["private_api_user"], config["private_api_password"] + ) try: - nb_deposits = requests.get('%s?page_size=1' % deposits_list_url, - auth=deposits_list_auth, - timeout=30).json()['count'] - - deposits_data = cache.get('swh-deposit-list') - if not deposits_data or deposits_data['count'] != nb_deposits: - deposits_data = requests.get('%s?page_size=%s' % - (deposits_list_url, nb_deposits), - auth=deposits_list_auth, - timeout=30).json() - cache.set('swh-deposit-list', deposits_data) - - deposits = deposits_data['results'] - - search_value = request.GET['search[value]'] + nb_deposits = requests.get( + "%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30 + ).json()["count"] + + deposits_data = cache.get("swh-deposit-list") + if not deposits_data or deposits_data["count"] != nb_deposits: + deposits_data = requests.get( + "%s?page_size=%s" % (deposits_list_url, nb_deposits), + auth=deposits_list_auth, + timeout=30, + ).json() + cache.set("swh-deposit-list", deposits_data) + + deposits = deposits_data["results"] + + search_value = request.GET["search[value]"] if search_value: - deposits = \ - [d for d in deposits - if any(search_value.lower() in val - for val in [str(v).lower() for v in d.values()])] - - column_order = request.GET['order[0][column]'] - field_order = request.GET['columns[%s][name]' % column_order] - order_dir = request.GET['order[0][dir]'] - - deposits = sorted(deposits, key=lambda d: d[field_order] or '') - if order_dir == 'desc': + deposits = [ + d + for d in deposits + if any( + search_value.lower() in val + for val in [str(v).lower() for v in d.values()] + ) + ] + + column_order = request.GET["order[0][column]"] + field_order = request.GET["columns[%s][name]" % column_order] + order_dir = request.GET["order[0][dir]"] + + deposits = sorted(deposits, key=lambda d: d[field_order] or "") + if order_dir == "desc": deposits = list(reversed(deposits)) - length = int(request.GET['length']) - page = int(request.GET['start']) / length + 1 + length = int(request.GET["length"]) + page = int(request.GET["start"]) / length + 1 paginator = Paginator(deposits, length) data = paginator.page(page).object_list - table_data['recordsTotal'] = deposits_data['count'] - table_data['recordsFiltered'] = len(deposits) - table_data['data'] = [{ - 'id': d['id'], - 'external_id': d['external_id'], - 'reception_date': d['reception_date'], - 'status': d['status'], - 'status_detail': d['status_detail'], - 'swh_anchor_id': d['swh_anchor_id'], - 'swh_anchor_id_context': d['swh_anchor_id_context'], - 'swh_id': d['swh_id'], - 'swh_id_context': d['swh_id_context'] - } for d in data] + table_data["recordsTotal"] = deposits_data["count"] + table_data["recordsFiltered"] = len(deposits) + table_data["data"] = [ + { + "id": d["id"], + "external_id": d["external_id"], + "reception_date": d["reception_date"], + "status": d["status"], + "status_detail": d["status_detail"], + "swh_anchor_id": d["swh_anchor_id"], + "swh_anchor_id_context": d["swh_anchor_id_context"], + "swh_id": d["swh_id"], + "swh_id_context": d["swh_id_context"], + } + for d in data + ] except Exception as exc: sentry_sdk.capture_exception(exc) - table_data['error'] = ('An error occurred while retrieving ' - 'the list of deposits !') + table_data["error"] = ( + "An error occurred while retrieving " "the list of deposits !" + ) - return HttpResponse(json.dumps(table_data), - content_type='application/json') + return HttpResponse(json.dumps(table_data), content_type="application/json") diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py index f84b0d4b..41a740d0 100644 --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -1,206 +1,229 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf import settings from django.contrib.admin.views.decorators import staff_member_required from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import Paginator from django.http import HttpResponse from django.shortcuts import render from django.views.decorators.http import require_POST from swh.web.admin.adminurls import admin_route from swh.web.common.models import ( - SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest + SaveAuthorizedOrigin, + SaveUnauthorizedOrigin, + SaveOriginRequest, ) from swh.web.common.origin_save import ( - create_save_origin_request, get_save_origin_task_info, - SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED + create_save_origin_request, + get_save_origin_task_info, + SAVE_REQUEST_PENDING, + SAVE_REQUEST_REJECTED, ) -@admin_route(r'origin/save/', view_name='admin-origin-save') +@admin_route(r"origin/save/", view_name="admin-origin-save") @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save(request): - return render(request, 'admin/origin-save.html') + return render(request, "admin/origin-save.html") def _datatables_origin_urls_response(request, urls_query_set): - search_value = request.GET['search[value]'] + search_value = request.GET["search[value]"] if search_value: urls_query_set = urls_query_set.filter(url__icontains=search_value) - column_order = request.GET['order[0][column]'] - field_order = request.GET['columns[%s][name]' % column_order] - order_dir = request.GET['order[0][dir]'] - if order_dir == 'desc': - field_order = '-' + field_order + column_order = request.GET["order[0][column]"] + field_order = request.GET["columns[%s][name]" % column_order] + order_dir = request.GET["order[0][dir]"] + if order_dir == "desc": + field_order = "-" + field_order urls_query_set = urls_query_set.order_by(field_order) table_data = {} - table_data['draw'] = int(request.GET['draw']) - table_data['recordsTotal'] = urls_query_set.count() - table_data['recordsFiltered'] = urls_query_set.count() - length = int(request.GET['length']) - page = int(request.GET['start']) / length + 1 + table_data["draw"] = int(request.GET["draw"]) + table_data["recordsTotal"] = urls_query_set.count() + table_data["recordsFiltered"] = urls_query_set.count() + length = int(request.GET["length"]) + page = int(request.GET["start"]) / length + 1 paginator = Paginator(urls_query_set, length) urls_query_set = paginator.page(page).object_list - table_data['data'] = [{'url': u.url} for u in urls_query_set] - table_data_json = json.dumps(table_data, separators=(',', ': ')) - return HttpResponse(table_data_json, content_type='application/json') + table_data["data"] = [{"url": u.url} for u in urls_query_set] + table_data_json = json.dumps(table_data, separators=(",", ": ")) + return HttpResponse(table_data_json, content_type="application/json") -@admin_route(r'origin/save/authorized_urls/list/', - view_name='admin-origin-save-authorized-urls-list') +@admin_route( + r"origin/save/authorized_urls/list/", + view_name="admin-origin-save-authorized-urls-list", +) @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_authorized_urls_list(request): authorized_urls = SaveAuthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, authorized_urls) -@admin_route(r'origin/save/authorized_urls/add/(?P.+)/', - view_name='admin-origin-save-add-authorized-url') +@admin_route( + r"origin/save/authorized_urls/add/(?P.+)/", + view_name="admin-origin-save-add-authorized-url", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_add_authorized_url(request, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: # add the new authorized url SaveAuthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist - pending_save_requests = \ - SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, - status=SAVE_REQUEST_PENDING) + pending_save_requests = SaveOriginRequest.objects.filter( + origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING + ) # create origin save tasks for previously pending requests for psr in pending_save_requests: create_save_origin_request(psr.visit_type, psr.origin_url) status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) -@admin_route(r'origin/save/authorized_urls/remove/(?P.+)/', - view_name='admin-origin-save-remove-authorized-url') +@admin_route( + r"origin/save/authorized_urls/remove/(?P.+)/", + view_name="admin-origin-save-remove-authorized-url", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_remove_authorized_url(request, origin_url): try: entry = SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) -@admin_route(r'origin/save/unauthorized_urls/list/', - view_name='admin-origin-save-unauthorized-urls-list') +@admin_route( + r"origin/save/unauthorized_urls/list/", + view_name="admin-origin-save-unauthorized-urls-list", +) @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_unauthorized_urls_list(request): unauthorized_urls = SaveUnauthorizedOrigin.objects.all() return _datatables_origin_urls_response(request, unauthorized_urls) -@admin_route(r'origin/save/unauthorized_urls/add/(?P.+)/', - view_name='admin-origin-save-add-unauthorized-url') +@admin_route( + r"origin/save/unauthorized_urls/add/(?P.+)/", + view_name="admin-origin-save-add-unauthorized-url", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_add_unauthorized_url(request, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) # check if pending save requests with that url prefix exist - pending_save_requests = \ - SaveOriginRequest.objects.filter(origin_url__startswith=origin_url, - status=SAVE_REQUEST_PENDING) + pending_save_requests = SaveOriginRequest.objects.filter( + origin_url__startswith=origin_url, status=SAVE_REQUEST_PENDING + ) # mark pending requests as rejected for psr in pending_save_requests: psr.status = SAVE_REQUEST_REJECTED psr.save() status_code = 200 else: status_code = 400 return HttpResponse(status=status_code) -@admin_route(r'origin/save/unauthorized_urls/remove/(?P.+)/', - view_name='admin-origin-save-remove-unauthorized-url') +@admin_route( + r"origin/save/unauthorized_urls/remove/(?P.+)/", + view_name="admin-origin-save-remove-unauthorized-url", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_remove_unauthorized_url(request, origin_url): try: entry = SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) -@admin_route(r'origin/save/request/accept/(?P.+)/url/(?P.+)/', # noqa - view_name='admin-origin-save-request-accept') +@admin_route( + r"origin/save/request/accept/(?P.+)/url/(?P.+)/", + view_name="admin-origin-save-request-accept", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_request_accept(request, visit_type, origin_url): try: SaveAuthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveAuthorizedOrigin.objects.create(url=origin_url) create_save_origin_request(visit_type, origin_url) return HttpResponse(status=200) -@admin_route(r'origin/save/request/reject/(?P.+)/url/(?P.+)/', # noqa - view_name='admin-origin-save-request-reject') +@admin_route( + r"origin/save/request/reject/(?P.+)/url/(?P.+)/", + view_name="admin-origin-save-request-reject", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_request_reject(request, visit_type, origin_url): try: SaveUnauthorizedOrigin.objects.get(url=origin_url) except ObjectDoesNotExist: SaveUnauthorizedOrigin.objects.create(url=origin_url) - sor = SaveOriginRequest.objects.get(visit_type=visit_type, - origin_url=origin_url, - status=SAVE_REQUEST_PENDING) + sor = SaveOriginRequest.objects.get( + visit_type=visit_type, origin_url=origin_url, status=SAVE_REQUEST_PENDING + ) sor.status = SAVE_REQUEST_REJECTED sor.save() return HttpResponse(status=200) -@admin_route(r'origin/save/request/remove/(?P.+)/', - view_name='admin-origin-save-request-remove') +@admin_route( + r"origin/save/request/remove/(?P.+)/", + view_name="admin-origin-save-request-remove", +) @require_POST @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _admin_origin_save_request_remove(request, sor_id): try: entry = SaveOriginRequest.objects.get(id=sor_id) except ObjectDoesNotExist: status_code = 404 else: entry.delete() status_code = 200 return HttpResponse(status=status_code) -@admin_route(r'origin/save/task/info/(?P.+)/', - view_name='admin-origin-save-task-info') +@admin_route( + r"origin/save/task/info/(?P.+)/", + view_name="admin-origin-save-task-info", +) @staff_member_required(view_func=None, login_url=settings.LOGIN_URL) def _save_origin_task_info(request, save_request_id): request_info = get_save_origin_task_info(save_request_id) - for date_field in ('scheduled', 'started', 'ended'): + for date_field in ("scheduled", "started", "ended"): if date_field in request_info and request_info[date_field] is not None: request_info[date_field] = request_info[date_field].isoformat() - return HttpResponse(json.dumps(request_info), - content_type='application/json') + return HttpResponse(json.dumps(request_info), content_type="application/json") diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py index 008062a9..af8f2521 100644 --- a/swh/web/admin/urls.py +++ b/swh/web/admin/urls.py @@ -1,27 +1,25 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url from django.contrib.auth.views import LoginView from django.shortcuts import redirect from swh.web.admin.adminurls import AdminUrls -import swh.web.admin.origin_save # noqa -import swh.web.admin.deposit # noqa +import swh.web.admin.origin_save # noqa +import swh.web.admin.deposit # noqa def _admin_default_view(request): - return redirect('admin-origin-save') + return redirect("admin-origin-save") urlpatterns = [ - url(r'^$', _admin_default_view, name='admin'), - url(r'^login/$', - LoginView.as_view(template_name='login.html'), - name='login'), + url(r"^$", _admin_default_view, name="admin"), + url(r"^login/$", LoginView.as_view(template_name="login.html"), name="login"), ] urlpatterns += AdminUrls.get_url_patterns() diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py index d9a51fb1..2cc48fc1 100644 --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -1,441 +1,465 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import defaultdict import functools from functools import wraps import os import re import textwrap from typing import List import docutils.nodes import docutils.parsers.rst import docutils.utils from rest_framework.decorators import api_view import sentry_sdk from swh.web.common.utils import parse_rst from swh.web.api.apiurls import APIUrls from swh.web.api.apiresponse import make_api_response, error_response class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): """ docutils visitor for walking on a parsed rst document containing sphinx httpdomain roles. Its purpose is to extract relevant info regarding swh api endpoints (for instance url arguments) from their docstring written using sphinx httpdomain. """ # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) - parameter_roles = ('param', 'parameter', 'arg', 'argument') + parameter_roles = ("param", "parameter", "arg", "argument") - request_json_object_roles = ('reqjsonobj', 'reqjson', 'jsonobj', '>json') + response_json_object_roles = ("resjsonobj", "resjson", ">jsonobj", ">json") - response_json_array_roles = ('resjsonarr', '>jsonarr') + response_json_array_roles = ("resjsonarr", ">jsonarr") - query_parameter_roles = ('queryparameter', 'queryparam', 'qparam', 'query') + query_parameter_roles = ("queryparameter", "queryparam", "qparam", "query") - request_header_roles = ('header', 'resheader', 'responseheader') + response_header_roles = (">header", "resheader", "responseheader") - status_code_roles = ('statuscode', 'status', 'code') + status_code_roles = ("statuscode", "status", "code") def __init__(self, document, data): super().__init__(document) self.data = data self.args_set = set() self.params_set = set() self.inputs_set = set() self.returns_set = set() self.status_codes_set = set() self.reqheaders_set = set() self.resheaders_set = set() self.field_list_visited = False self.current_json_obj = None def process_paragraph(self, par): """ Process extracted paragraph text before display. Cleanup document model markups and transform the paragraph into a valid raw rst string (as the apidoc documentation transform rst to html when rendering). """ - par = par.replace('\n', ' ') + par = par.replace("\n", " ") # keep emphasized, strong and literal text - par = par.replace('', '*') - par = par.replace('', '*') - par = par.replace('', '**') - par = par.replace('', '**') - par = par.replace('', '``') - par = par.replace('', '``') + par = par.replace("", "*") + par = par.replace("", "*") + par = par.replace("", "**") + par = par.replace("", "**") + par = par.replace("", "``") + par = par.replace("", "``") # keep links to web pages - if '', - r'`\1 <\2>`_', par) + if "', + r"`\1 <\2>`_", + par, + ) # remove parsed document markups but keep rst links - par = re.sub(r'<[^<]+?>(?!`_)', '', par) + par = re.sub(r"<[^<]+?>(?!`_)", "", par) # api urls cleanup to generate valid links afterwards subs_made = 1 while subs_made: - (par, subs_made) = re.subn(r'(:http:.*)(\(\w+\))', r'\1', par) + (par, subs_made) = re.subn(r"(:http:.*)(\(\w+\))", r"\1", par) subs_made = 1 while subs_made: - (par, subs_made) = re.subn(r'(:http:.*)(\[.*\])', r'\1', par) - par = par.replace('//', '/') + (par, subs_made) = re.subn(r"(:http:.*)(\[.*\])", r"\1", par) + par = par.replace("//", "/") # transform references to api endpoints doc into valid rst links - par = re.sub(':http:get:`([^,`]*)`', r'`\1 <\1doc/>`_', par) + par = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", par) # transform references to some elements into bold text - par = re.sub(':http:header:`(.*)`', r'**\1**', par) - par = re.sub(':func:`(.*)`', r'**\1**', par) + par = re.sub(":http:header:`(.*)`", r"**\1**", par) + par = re.sub(":func:`(.*)`", r"**\1**", par) return par def visit_field_list(self, node): """ Visit parsed rst field lists to extract relevant info regarding api endpoint. """ self.field_list_visited = True for child in node.traverse(): # get the parsed field name if isinstance(child, docutils.nodes.field_name): field_name = child.astext() # parse field text elif isinstance(child, docutils.nodes.paragraph): text = self.process_paragraph(str(child)) - field_data = field_name.split(' ') + field_data = field_name.split(" ") # Parameters if field_data[0] in self.parameter_roles: if field_data[2] not in self.args_set: - self.data['args'].append({'name': field_data[2], - 'type': field_data[1], - 'doc': text}) + self.data["args"].append( + {"name": field_data[2], "type": field_data[1], "doc": text} + ) self.args_set.add(field_data[2]) # Query Parameters if field_data[0] in self.query_parameter_roles: if field_data[2] not in self.params_set: - self.data['params'].append({'name': field_data[2], - 'type': field_data[1], - 'doc': text}) + self.data["params"].append( + {"name": field_data[2], "type": field_data[1], "doc": text} + ) self.params_set.add(field_data[2]) # Request data type - if (field_data[0] in self.request_json_array_roles or - field_data[0] in self.request_json_object_roles): + if ( + field_data[0] in self.request_json_array_roles + or field_data[0] in self.request_json_object_roles + ): # array if field_data[0] in self.request_json_array_roles: - self.data['input_type'] = 'array' + self.data["input_type"] = "array" # object else: - self.data['input_type'] = 'object' + self.data["input_type"] = "object" # input object field if field_data[2] not in self.inputs_set: - self.data['inputs'].append({'name': field_data[2], - 'type': field_data[1], - 'doc': text}) + self.data["inputs"].append( + {"name": field_data[2], "type": field_data[1], "doc": text} + ) self.inputs_set.add(field_data[2]) - self.current_json_obj = self.data['inputs'][-1] + self.current_json_obj = self.data["inputs"][-1] # Response type - if (field_data[0] in self.response_json_array_roles or - field_data[0] in self.response_json_object_roles): + if ( + field_data[0] in self.response_json_array_roles + or field_data[0] in self.response_json_object_roles + ): # array if field_data[0] in self.response_json_array_roles: - self.data['return_type'] = 'array' + self.data["return_type"] = "array" # object else: - self.data['return_type'] = 'object' + self.data["return_type"] = "object" # returned object field if field_data[2] not in self.returns_set: - self.data['returns'].append({'name': field_data[2], - 'type': field_data[1], - 'doc': text}) + self.data["returns"].append( + {"name": field_data[2], "type": field_data[1], "doc": text} + ) self.returns_set.add(field_data[2]) - self.current_json_obj = self.data['returns'][-1] + self.current_json_obj = self.data["returns"][-1] # Status Codes if field_data[0] in self.status_code_roles: if field_data[1] not in self.status_codes_set: - self.data['status_codes'].append({'code': field_data[1], # noqa - 'doc': text}) + self.data["status_codes"].append( + {"code": field_data[1], "doc": text} + ) self.status_codes_set.add(field_data[1]) # Request Headers if field_data[0] in self.request_header_roles: if field_data[1] not in self.reqheaders_set: - self.data['reqheaders'].append({'name': field_data[1], - 'doc': text}) + self.data["reqheaders"].append( + {"name": field_data[1], "doc": text} + ) self.reqheaders_set.add(field_data[1]) # Response Headers if field_data[0] in self.response_header_roles: if field_data[1] not in self.resheaders_set: - resheader = {'name': field_data[1], - 'doc': text} - self.data['resheaders'].append(resheader) + resheader = {"name": field_data[1], "doc": text} + self.data["resheaders"].append(resheader) self.resheaders_set.add(field_data[1]) - if resheader['name'] == 'Content-Type' and \ - resheader['doc'] == 'application/octet-stream': - self.data['return_type'] = 'octet stream' + if ( + resheader["name"] == "Content-Type" + and resheader["doc"] == "application/octet-stream" + ): + self.data["return_type"] = "octet stream" def visit_paragraph(self, node): """ Visit relevant paragraphs to parse """ # only parsed top level paragraphs if isinstance(node.parent, docutils.nodes.block_quote): text = self.process_paragraph(str(node)) # endpoint description - if (not text.startswith('**') and - text not in self.data['description']): - self.data['description'] += '\n\n' if self.data['description'] else '' # noqa - self.data['description'] += text + if not text.startswith("**") and text not in self.data["description"]: + self.data["description"] += "\n\n" if self.data["description"] else "" + self.data["description"] += text def visit_literal_block(self, node): """ Visit literal blocks """ text = node.astext() # literal block in endpoint description if not self.field_list_visited: - self.data['description'] += \ - ':\n\n%s\n' % textwrap.indent(text, '\t') + self.data["description"] += ":\n\n%s\n" % textwrap.indent(text, "\t") # extract example url - if ':swh_web_api:' in text: - self.data['examples'].append( - '/api/1/' + re.sub('.*`(.*)`.*', r'\1', text)) + if ":swh_web_api:" in text: + self.data["examples"].append("/api/1/" + re.sub(".*`(.*)`.*", r"\1", text)) def visit_bullet_list(self, node): # bullet list in endpoint description if not self.field_list_visited: - self.data['description'] += '\n\n' + self.data["description"] += "\n\n" for child in node.traverse(): # process list item if isinstance(child, docutils.nodes.paragraph): line_text = self.process_paragraph(str(child)) - self.data['description'] += '\t* %s\n' % line_text + self.data["description"] += "\t* %s\n" % line_text elif self.current_json_obj: - self.current_json_obj['doc'] += '\n\n' + self.current_json_obj["doc"] += "\n\n" for child in node.traverse(): # process list item if isinstance(child, docutils.nodes.paragraph): line_text = self.process_paragraph(str(child)) - self.current_json_obj['doc'] += '\t\t* %s\n' % line_text + self.current_json_obj["doc"] += "\t\t* %s\n" % line_text self.current_json_obj = None def visit_warning(self, node): text = self.process_paragraph(str(node)) - rst_warning = '\n\n.. warning::\n%s\n' % textwrap.indent(text, '\t') - if rst_warning not in self.data['description']: - self.data['description'] += rst_warning + rst_warning = "\n\n.. warning::\n%s\n" % textwrap.indent(text, "\t") + if rst_warning not in self.data["description"]: + self.data["description"] += rst_warning def unknown_visit(self, node): pass def unknown_departure(self, node): pass def _parse_httpdomain_doc(doc, data): - doc_lines = doc.split('\n') + doc_lines = doc.split("\n") doc_lines_filtered = [] urls = defaultdict(list) - default_http_methods = ['HEAD', 'OPTIONS'] + default_http_methods = ["HEAD", "OPTIONS"] # httpdomain is a sphinx extension that is unknown to docutils but # fortunately we can still parse its directives' content, # so remove lines with httpdomain directives before executing the # rst parser from docutils for doc_line in doc_lines: - if '.. http' not in doc_line: + if ".. http" not in doc_line: doc_lines_filtered.append(doc_line) else: - url = doc_line[doc_line.find('/'):] + url = doc_line[doc_line.find("/") :] # emphasize url arguments for html rendering - url = re.sub(r'\((\w+)\)', r' **\(\1\)** ', url) - method = re.search(r'http:(\w+)::', doc_line).group(1) + url = re.sub(r"\((\w+)\)", r" **\(\1\)** ", url) + method = re.search(r"http:(\w+)::", doc_line).group(1) urls[url].append(method.upper()) for url, methods in urls.items(): - data['urls'].append({'rule': url, - 'methods': methods + default_http_methods}) + data["urls"].append({"rule": url, "methods": methods + default_http_methods}) # parse the rst docstring and do not print system messages about # unknown httpdomain roles - document = parse_rst('\n'.join(doc_lines_filtered), report_level=5) + document = parse_rst("\n".join(doc_lines_filtered), report_level=5) # remove the system_message nodes from the parsed document for node in document.traverse(docutils.nodes.system_message): node.parent.remove(node) # visit the document nodes to extract relevant endpoint info visitor = _HTTPDomainDocVisitor(document, data) document.walkabout(visitor) class APIDocException(Exception): """ Custom exception to signal errors in the use of the APIDoc decorators """ -def api_doc(route: str, noargs: bool = False, need_params: bool = False, - tags: List[str] = [], handle_response: bool = False, - api_version: str = '1'): +def api_doc( + route: str, + noargs: bool = False, + need_params: bool = False, + tags: List[str] = [], + handle_response: bool = False, + api_version: str = "1", +): """ Decorator for an API endpoint implementation used to generate a dedicated view displaying its HTML documentation. The documentation will be generated from the endpoint docstring based on sphinxcontrib-httpdomain format. Args: route: documentation page's route noargs: set to True if the route has no arguments, and its result should be displayed anytime its documentation is requested. Default to False need_params: specify the route requires query parameters otherwise errors will occur. It enables to avoid displaying the invalid response in its HTML documentation. Default to False. tags: Further information on api endpoints. Two values are possibly expected: * hidden: remove the entry points from the listing * upcoming: display the entry point but it is not followable handle_response: indicate if the decorated function takes care of creating the HTTP response or delegates that task to the apiresponse module api_version: api version string """ tags_set = set(tags) # @api_doc() Decorator call def decorator(f): # if the route is not hidden, add it to the index - if 'hidden' not in tags_set: + if "hidden" not in tags_set: doc_data = get_doc_data(f, route, noargs) - doc_desc = doc_data['description'] - first_dot_pos = doc_desc.find('.') - APIUrls.add_doc_route(route, doc_desc[:first_dot_pos+1], - noargs=noargs, api_version=api_version, - tags=tags_set) + doc_desc = doc_data["description"] + first_dot_pos = doc_desc.find(".") + APIUrls.add_doc_route( + route, + doc_desc[: first_dot_pos + 1], + noargs=noargs, + api_version=api_version, + tags=tags_set, + ) # create a dedicated view to display endpoint HTML doc - @api_view(['GET', 'HEAD']) + @api_view(["GET", "HEAD"]) @wraps(f) def doc_view(request): doc_data = get_doc_data(f, route, noargs) return make_api_response(request, None, doc_data) - route_name = '%s-doc' % route[1:-1].replace('/', '-') - urlpattern = f'^{api_version}{route}doc/$' + route_name = "%s-doc" % route[1:-1].replace("/", "-") + urlpattern = f"^{api_version}{route}doc/$" - view_name = 'api-%s-%s' % (api_version, route_name) + view_name = "api-%s-%s" % (api_version, route_name) APIUrls.add_url_pattern(urlpattern, doc_view, view_name) @wraps(f) def documented_view(request, **kwargs): doc_data = get_doc_data(f, route, noargs) try: response = f(request, **kwargs) except Exception as exc: sentry_sdk.capture_exception(exc) - if request.accepted_media_type == 'text/html' and \ - need_params and not request.query_params: + if ( + request.accepted_media_type == "text/html" + and need_params + and not request.query_params + ): response = None else: return error_response(request, exc, doc_data) if handle_response: return response else: return make_api_response(request, response, doc_data) return documented_view return decorator @functools.lru_cache(maxsize=32) def get_doc_data(f, route, noargs): """ Build documentation data for the decorated api endpoint function """ data = { - 'description': '', - 'response_data': None, - 'urls': [], - 'args': [], - 'params': [], - 'input_type': '', - 'inputs': [], - 'resheaders': [], - 'reqheaders': [], - 'return_type': '', - 'returns': [], - 'status_codes': [], - 'examples': [], - 'route': route, - 'noargs': noargs + "description": "", + "response_data": None, + "urls": [], + "args": [], + "params": [], + "input_type": "", + "inputs": [], + "resheaders": [], + "reqheaders": [], + "return_type": "", + "returns": [], + "status_codes": [], + "examples": [], + "route": route, + "noargs": noargs, } if not f.__doc__: - raise APIDocException('apidoc: expected a docstring' - ' for function %s' - % (f.__name__,)) + raise APIDocException( + "apidoc: expected a docstring" " for function %s" % (f.__name__,) + ) # use raw docstring as endpoint documentation if sphinx # httpdomain is not used - if '.. http' not in f.__doc__: - data['description'] = f.__doc__ + if ".. http" not in f.__doc__: + data["description"] = f.__doc__ # else parse the sphinx httpdomain docstring with docutils # (except when building the swh-web documentation through autodoc # sphinx extension, not needed and raise errors with sphinx >= 1.7) - elif 'SWH_WEB_DOC_BUILD' not in os.environ: + elif "SWH_WEB_DOC_BUILD" not in os.environ: _parse_httpdomain_doc(f.__doc__, data) # process input/returned object info for nicer html display - inputs_list = '' - returns_list = '' - for inp in data['inputs']: + inputs_list = "" + returns_list = "" + for inp in data["inputs"]: # special case for array of non object type, for instance # :jsonarr string -: an array of string - if ret['name'] != '-': - returns_list += ('\t* **%s (%s)**: %s\n' % - (ret['name'], ret['type'], ret['doc'])) - data['inputs_list'] = inputs_list - data['returns_list'] = returns_list + if ret["name"] != "-": + returns_list += "\t* **%s (%s)**: %s\n" % ( + ret["name"], + ret["type"], + ret["doc"], + ) + data["inputs_list"] = inputs_list + data["returns_list"] = returns_list return data -DOC_COMMON_HEADERS = ''' +DOC_COMMON_HEADERS = """ :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` - header of request''' -DOC_RESHEADER_LINK = ''' + header of request""" +DOC_RESHEADER_LINK = """ :resheader Link: indicates that a subsequent result page is available and contains the url pointing to it -''' +""" DEFAULT_SUBSTITUTIONS = { - 'common_headers': DOC_COMMON_HEADERS, - 'resheader_link': DOC_RESHEADER_LINK, + "common_headers": DOC_COMMON_HEADERS, + "resheader_link": DOC_RESHEADER_LINK, } def format_docstring(**substitutions): def decorator(f): - f.__doc__ = f.__doc__.format(**{ - **DEFAULT_SUBSTITUTIONS, **substitutions}) + f.__doc__ = f.__doc__.format(**{**DEFAULT_SUBSTITUTIONS, **substitutions}) return f + return decorator diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py index 773cda44..ab2d27ab 100644 --- a/swh/web/api/apiresponse.py +++ b/swh/web/api/apiresponse.py @@ -1,196 +1,190 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import traceback from django.utils.html import escape from rest_framework.response import Response from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api import utils -from swh.web.common.exc import ( - NotFoundExc, ForbiddenExc, - BadInputExc, LargePayloadExc -) +from swh.web.common.exc import NotFoundExc, ForbiddenExc, BadInputExc, LargePayloadExc from swh.web.common.utils import shorten_path, gen_path_info from swh.web.config import get_config def compute_link_header(rv, options): """Add Link header in returned value results. Args: request: a DRF Request object rv (dict): dictionary with keys: - headers: potential headers with 'link-next' and 'link-prev' keys - results: containing the result to return options (dict): the initial dict to update with result if any Returns: dict: dictionary with optional keys 'link-next' and 'link-prev' """ link_headers = [] - if 'headers' not in rv: + if "headers" not in rv: return {} - rv_headers = rv['headers'] + rv_headers = rv["headers"] - if 'link-next' in rv_headers: - link_headers.append('<%s>; rel="next"' % rv_headers['link-next']) - if 'link-prev' in rv_headers: - link_headers.append('<%s>; rel="previous"' % rv_headers['link-prev']) + if "link-next" in rv_headers: + link_headers.append('<%s>; rel="next"' % rv_headers["link-next"]) + if "link-prev" in rv_headers: + link_headers.append('<%s>; rel="previous"' % rv_headers["link-prev"]) if link_headers: - link_header_str = ','.join(link_headers) - headers = options.get('headers', {}) - headers.update({ - 'Link': link_header_str - }) + link_header_str = ",".join(link_headers) + headers = options.get("headers", {}) + headers.update({"Link": link_header_str}) return headers return {} def filter_by_fields(request, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ - fields = request.query_params.get('fields') + fields = request.query_params.get("fields") if fields: - fields = set(fields.split(',')) + fields = set(fields.split(",")) data = utils.filter_field_keys(data, fields) return data def transform(rv): """Transform an eventual returned value with multiple layer of information with only what's necessary. If the returned value rv contains the 'results' key, this is the associated value which is returned. Otherwise, return the initial dict without the potential 'headers' key. """ - if 'results' in rv: - return rv['results'] + if "results" in rv: + return rv["results"] - if 'headers' in rv: - rv.pop('headers') + if "headers" in rv: + rv.pop("headers") return rv def make_api_response(request, data, doc_data={}, options={}): """Generates an API response based on the requested mimetype. Args: request: a DRF Request object data: raw data to return in the API response doc_data: documentation data for HTML response options: optional data that can be used to generate the response Returns: a DRF Response a object """ if data: - options['headers'] = compute_link_header(data, options) + options["headers"] = compute_link_header(data, options) data = transform(data) data = filter_by_fields(request, data) doc_env = doc_data headers = {} - if 'headers' in options: - doc_env['headers_data'] = options['headers'] - headers = options['headers'] + if "headers" in options: + doc_env["headers_data"] = options["headers"] + headers = options["headers"] # get request status code - doc_env['status_code'] = options.get('status', 200) + doc_env["status_code"] = options.get("status", 200) - response_args = {'status': doc_env['status_code'], - 'headers': headers, - 'content_type': request.accepted_media_type} + response_args = { + "status": doc_env["status_code"], + "headers": headers, + "content_type": request.accepted_media_type, + } # when requesting HTML, typically when browsing the API through its # documented views, we need to enrich the input data with documentation # related ones and inform DRF that we request HTML template rendering - if request.accepted_media_type == 'text/html': + if request.accepted_media_type == "text/html": if data: - data = json.dumps(data, sort_keys=True, - indent=4, - separators=(',', ': ')) - doc_env['response_data'] = data - doc_env['heading'] = shorten_path(str(request.path)) + data = json.dumps(data, sort_keys=True, indent=4, separators=(",", ": ")) + doc_env["response_data"] = data + doc_env["heading"] = shorten_path(str(request.path)) # generate breadcrumbs data - if 'route' in doc_env: - doc_env['endpoint_path'] = gen_path_info(doc_env['route']) - for i in range(len(doc_env['endpoint_path']) - 1): - doc_env['endpoint_path'][i]['path'] += '/doc/' - if not doc_env['noargs']: - doc_env['endpoint_path'][-1]['path'] += '/doc/' + if "route" in doc_env: + doc_env["endpoint_path"] = gen_path_info(doc_env["route"]) + for i in range(len(doc_env["endpoint_path"]) - 1): + doc_env["endpoint_path"][i]["path"] += "/doc/" + if not doc_env["noargs"]: + doc_env["endpoint_path"][-1]["path"] += "/doc/" - response_args['data'] = doc_env - response_args['template_name'] = 'api/apidoc.html' + response_args["data"] = doc_env + response_args["template_name"] = "api/apidoc.html" # otherwise simply return the raw data and let DRF picks # the correct renderer (JSON or YAML) else: - response_args['data'] = data + response_args["data"] = data return Response(**response_args) def error_response(request, error, doc_data): """Private function to create a custom error response. Args: request: a DRF Request object error: the exception that caused the error doc_data: documentation data for HTML response """ error_code = 500 if isinstance(error, BadInputExc): error_code = 400 elif isinstance(error, NotFoundExc): error_code = 404 elif isinstance(error, ForbiddenExc): error_code = 403 elif isinstance(error, LargePayloadExc): error_code = 413 elif isinstance(error, StorageDBError): error_code = 503 elif isinstance(error, StorageAPIError): error_code = 503 - error_opts = {'status': error_code} + error_opts = {"status": error_code} error_data = { - 'exception': error.__class__.__name__, - 'reason': str(error), + "exception": error.__class__.__name__, + "reason": str(error), } - if request.accepted_media_type == 'text/html': - error_data['reason'] = escape(error_data['reason']) + if request.accepted_media_type == "text/html": + error_data["reason"] = escape(error_data["reason"]) - if get_config()['debug']: - error_data['traceback'] = traceback.format_exc() + if get_config()["debug"]: + error_data["traceback"] = traceback.format_exc() - return make_api_response(request, error_data, doc_data, - options=error_opts) + return make_api_response(request, error_data, doc_data, options=error_opts) diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py index 9b037ead..1810ca41 100644 --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -1,91 +1,96 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from typing import Dict from rest_framework.decorators import api_view from swh.web.common.urlsindex import UrlsIndex from swh.web.api import throttling class APIUrls(UrlsIndex): """ Class to manage API documentation URLs. - Indexes all routes documented using apidoc's decorators. - Tracks endpoint/request processing method relationships for use in generating related urls in API documentation """ + _apidoc_routes = {} # type: Dict[str, Dict[str, str]] - scope = 'api' + scope = "api" @classmethod def get_app_endpoints(cls): return cls._apidoc_routes @classmethod - def add_doc_route(cls, route, docstring, noargs=False, - api_version='1', **kwargs): + def add_doc_route(cls, route, docstring, noargs=False, api_version="1", **kwargs): """ Add a route to the self-documenting API reference """ - route_name = route[1:-1].replace('/', '-') + route_name = route[1:-1].replace("/", "-") if not noargs: - route_name = '%s-doc' % route_name - route_view_name = 'api-%s-%s' % (api_version, route_name) + route_name = "%s-doc" % route_name + route_view_name = "api-%s-%s" % (api_version, route_name) if route not in cls._apidoc_routes: - d = {'docstring': docstring, - 'route': '/api/%s%s' % (api_version, route), - 'route_view_name': route_view_name} + d = { + "docstring": docstring, + "route": "/api/%s%s" % (api_version, route), + "route_view_name": route_view_name, + } for k, v in kwargs.items(): d[k] = v cls._apidoc_routes[route] = d -def api_route(url_pattern=None, view_name=None, - methods=['GET', 'HEAD', 'OPTIONS'], - throttle_scope='swh_api', - api_version='1', - checksum_args=None): +def api_route( + url_pattern=None, + view_name=None, + methods=["GET", "HEAD", "OPTIONS"], + throttle_scope="swh_api", + api_version="1", + checksum_args=None, +): """ Decorator to ease the registration of an API endpoint using the Django REST Framework. Args: url_pattern: the url pattern used by DRF to identify the API route view_name: the name of the API view associated to the route used to reverse the url methods: array of HTTP methods supported by the API route """ - url_pattern = '^' + api_version + url_pattern + '$' + url_pattern = "^" + api_version + url_pattern + "$" def decorator(f): # create a DRF view from the wrapped function @api_view(methods) @throttling.throttle_scope(throttle_scope) @functools.wraps(f) def api_view_f(*args, **kwargs): return f(*args, **kwargs) + # small hacks for correctly generating API endpoints index doc api_view_f.__name__ = f.__name__ api_view_f.http_method_names = methods # register the route and its view in the endpoints index - APIUrls.add_url_pattern(url_pattern, api_view_f, - view_name) + APIUrls.add_url_pattern(url_pattern, api_view_f, view_name) if checksum_args: - APIUrls.add_redirect_for_checksum_args(view_name, - [url_pattern], - checksum_args) + APIUrls.add_redirect_for_checksum_args( + view_name, [url_pattern], checksum_args + ) return f return decorator diff --git a/swh/web/api/renderers.py b/swh/web/api/renderers.py index 689091f4..bd1f5af9 100644 --- a/swh/web/api/renderers.py +++ b/swh/web/api/renderers.py @@ -1,37 +1,37 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import yaml from rest_framework import renderers class YAMLRenderer(renderers.BaseRenderer): """ Renderer which serializes to YAML. """ - media_type = 'application/yaml' - format = 'yaml' - charset = 'utf-8' + media_type = "application/yaml" + format = "yaml" + charset = "utf-8" ensure_ascii = False default_flow_style = False def render(self, data, accepted_media_type=None, renderer_context=None): """ Renders `data` into serialized YAML. """ - assert yaml, 'YAMLRenderer requires pyyaml to be installed' + assert yaml, "YAMLRenderer requires pyyaml to be installed" if data is None: - return '' + return "" return yaml.dump( data, stream=None, encoding=self.charset, allow_unicode=not self.ensure_ascii, - default_flow_style=self.default_flow_style + default_flow_style=self.default_flow_style, ) diff --git a/swh/web/api/throttling.py b/swh/web/api/throttling.py index e27bb2a8..2f7a80e7 100644 --- a/swh/web/api/throttling.py +++ b/swh/web/api/throttling.py @@ -1,162 +1,164 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from ipaddress import ( - ip_address, ip_network, IPv4Network, IPv6Network -) +from ipaddress import ip_address, ip_network, IPv4Network, IPv6Network from typing import Callable, List, TypeVar, Union from django.core.exceptions import ImproperlyConfigured import rest_framework from rest_framework.throttling import ScopedRateThrottle import sentry_sdk from swh.web.config import get_config -APIView = TypeVar('APIView', bound='rest_framework.views.APIView') +APIView = TypeVar("APIView", bound="rest_framework.views.APIView") Request = rest_framework.request.Request class SwhWebRateThrottle(ScopedRateThrottle): """Custom request rate limiter for DRF enabling to exempt specific networks specified in swh-web configuration. Requests are grouped into scopes. It enables to apply different requests rate limiting based on the scope name but also the input HTTP request types. To associate a scope to requests, one must add a 'throttle_scope' attribute when using a class based view, or call the 'throttle_scope' decorator when using a function based view. By default, requests do not have an associated scope and are not rate limited. Rate limiting can also be configured according to the type of the input HTTP requests for fine grained tuning. For instance, the following YAML configuration section sets a rate of: - 1 per minute for POST requests - 60 per minute for other request types for the 'swh_api' scope while exempting those coming from the 127.0.0.0/8 ip network. .. code-block:: yaml throttling: scopes: swh_api: limiter_rate: default: 60/m POST: 1/m exempted_networks: - 127.0.0.0/8 """ scope = None def __init__(self): super().__init__() self.exempted_networks = None - def get_exempted_networks(self, scope_name: str - ) -> List[Union[IPv4Network, IPv6Network]]: + def get_exempted_networks( + self, scope_name: str + ) -> List[Union[IPv4Network, IPv6Network]]: if not self.exempted_networks: - scopes = get_config()['throttling']['scopes'] + scopes = get_config()["throttling"]["scopes"] scope = scopes.get(scope_name) if scope: - networks = scope.get('exempted_networks') + networks = scope.get("exempted_networks") if networks: - self.exempted_networks = [ip_network(network) - for network in networks] + self.exempted_networks = [ + ip_network(network) for network in networks + ] return self.exempted_networks def allow_request(self, request: Request, view: APIView) -> bool: # no throttling for staff users if request.user.is_authenticated and request.user.is_staff: return True # class based view case if not self.scope: default_scope = getattr(view, self.scope_attr, None) request_allowed = None if default_scope is not None: # check if there is a specific rate limiting associated # to the request type assert request.method is not None - request_scope = f'{default_scope}_{request.method.lower()}' + request_scope = f"{default_scope}_{request.method.lower()}" setattr(view, self.scope_attr, request_scope) try: request_allowed = super().allow_request(request, view) # use default rate limiting otherwise except ImproperlyConfigured as exc: sentry_sdk.capture_exception(exc) setattr(view, self.scope_attr, default_scope) if request_allowed is None: request_allowed = super().allow_request(request, view) # function based view case else: default_scope = self.scope # check if there is a specific rate limiting associated # to the request type - self.scope = default_scope + '_' + request.method.lower() + self.scope = default_scope + "_" + request.method.lower() try: self.rate = self.get_rate() # use default rate limiting otherwise except ImproperlyConfigured as exc: sentry_sdk.capture_exception(exc) self.scope = default_scope self.rate = self.get_rate() self.num_requests, self.duration = self.parse_rate(self.rate) - request_allowed = \ - super(ScopedRateThrottle, self).allow_request(request, view) + request_allowed = super(ScopedRateThrottle, self).allow_request( + request, view + ) self.scope = default_scope exempted_networks = self.get_exempted_networks(default_scope) exempted_ip = False if exempted_networks: remote_address = ip_address(self.get_ident(request)) - exempted_ip = any(remote_address in network - for network in exempted_networks) + exempted_ip = any( + remote_address in network for network in exempted_networks + ) request_allowed = exempted_ip or request_allowed # set throttling related data in the request metadata # in order for the ThrottlingHeadersMiddleware to # add X-RateLimit-* headers in the HTTP response - if not exempted_ip and hasattr(self, 'history'): + if not exempted_ip and hasattr(self, "history"): hit_count = len(self.history) - request.META['RateLimit-Limit'] = self.num_requests - request.META['RateLimit-Remaining'] = self.num_requests - hit_count + request.META["RateLimit-Limit"] = self.num_requests + request.META["RateLimit-Remaining"] = self.num_requests - hit_count wait = self.wait() if wait is not None: - request.META['RateLimit-Reset'] = int(self.now + wait) + request.META["RateLimit-Reset"] = int(self.now + wait) return request_allowed def throttle_scope(scope: str) -> Callable[..., APIView]: """Decorator that allows the throttle scope of a DRF function based view to be set:: @api_view(['GET', ]) @throttle_scope('scope') def view(request): ... """ + def decorator(func: APIView) -> APIView: SwhScopeRateThrottle = type( - 'CustomScopeRateThrottle', - (SwhWebRateThrottle,), - {'scope': scope} + "CustomScopeRateThrottle", (SwhWebRateThrottle,), {"scope": scope} ) - func.throttle_classes = (SwhScopeRateThrottle, ) + func.throttle_classes = (SwhScopeRateThrottle,) return func + return decorator diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py index 0659ba36..39e6167d 100644 --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -1,20 +1,20 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import swh.web.api.views.content # noqa -import swh.web.api.views.directory # noqa -import swh.web.api.views.identifiers # noqa -import swh.web.api.views.origin # noqa -import swh.web.api.views.origin_save # noqa -import swh.web.api.views.release # noqa -import swh.web.api.views.revision # noqa -import swh.web.api.views.snapshot # noqa -import swh.web.api.views.stat # noqa -import swh.web.api.views.vault # noqa -import swh.web.api.views.ping # noqa +import swh.web.api.views.content # noqa +import swh.web.api.views.directory # noqa +import swh.web.api.views.identifiers # noqa +import swh.web.api.views.origin # noqa +import swh.web.api.views.origin_save # noqa +import swh.web.api.views.release # noqa +import swh.web.api.views.revision # noqa +import swh.web.api.views.snapshot # noqa +import swh.web.api.views.stat # noqa +import swh.web.api.views.vault # noqa +import swh.web.api.views.ping # noqa from swh.web.api.apiurls import APIUrls urlpatterns = APIUrls.get_url_patterns() diff --git a/swh/web/api/utils.py b/swh/web/api/utils.py index 8c1009ed..f5096ad7 100644 --- a/swh/web/api/utils.py +++ b/swh/web/api/utils.py @@ -1,324 +1,335 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict, Optional, Any from django.http import HttpRequest from swh.web.common.utils import reverse, resolve_branch_alias from swh.web.common.query import parse_hash def filter_field_keys(data, field_keys): """Given an object instance (directory or list), and a csv field keys to filter on. Return the object instance with filtered keys. Note: Returns obj as is if it's an instance of types not in (dictionary, list) Args: - data: one object (dictionary, list...) to filter. - field_keys: csv or set of keys to filter the object on Returns: obj filtered on field_keys """ if isinstance(data, map): return map(lambda x: filter_field_keys(x, field_keys), data) if isinstance(data, list): return [filter_field_keys(x, field_keys) for x in data] if isinstance(data, dict): return {k: v for (k, v) in data.items() if k in field_keys} return data def person_to_string(person): """Map a person (person, committer, tagger, etc...) to a string. """ - return ''.join([person['name'], ' <', person['email'], '>']) + return "".join([person["name"], " <", person["email"], ">"]) -def enrich_object(object: Dict[str, str], - request: Optional[HttpRequest] = None) -> Dict[str, str]: +def enrich_object( + object: Dict[str, str], request: Optional[HttpRequest] = None +) -> Dict[str, str]: """Enrich an object (revision, release) with link to the 'target' of type 'target_type'. Args: object: An object with target and target_type keys (e.g. release, revision) request: Absolute URIs will be generated if provided Returns: Object enriched with target object url (revision, release, content, directory) """ - if 'target' in object and 'target_type' in object: - if object['target_type'] in ('revision', 'release', 'directory'): - object['target_url'] = reverse( - 'api-1-%s' % object['target_type'], - url_args={'sha1_git': object['target']}, - request=request) - elif object['target_type'] == 'content': - object['target_url'] = reverse( - 'api-1-content', - url_args={'q': 'sha1_git:' + object['target']}, - request=request) - elif object['target_type'] == 'snapshot': - object['target_url'] = reverse( - 'api-1-snapshot', - url_args={'snapshot_id': object['target']}, - request=request) + if "target" in object and "target_type" in object: + if object["target_type"] in ("revision", "release", "directory"): + object["target_url"] = reverse( + "api-1-%s" % object["target_type"], + url_args={"sha1_git": object["target"]}, + request=request, + ) + elif object["target_type"] == "content": + object["target_url"] = reverse( + "api-1-content", + url_args={"q": "sha1_git:" + object["target"]}, + request=request, + ) + elif object["target_type"] == "snapshot": + object["target_url"] = reverse( + "api-1-snapshot", + url_args={"snapshot_id": object["target"]}, + request=request, + ) return object enrich_release = enrich_object -def enrich_directory(directory: Dict[str, str], - request: Optional[HttpRequest] = None) -> Dict[str, str]: +def enrich_directory( + directory: Dict[str, str], request: Optional[HttpRequest] = None +) -> Dict[str, str]: """Enrich directory with url to content or directory. Args: directory: dict of data associated to a swh directory object request: Absolute URIs will be generated if provided Returns: An enriched directory dict filled with additional urls """ - if 'type' in directory: - target_type = directory['type'] - target = directory['target'] - if target_type == 'file': - directory['target_url'] = reverse( - 'api-1-content', - url_args={'q': 'sha1_git:%s' % target}, - request=request) - elif target_type == 'dir': - directory['target_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': target}, - request=request) + if "type" in directory: + target_type = directory["type"] + target = directory["target"] + if target_type == "file": + directory["target_url"] = reverse( + "api-1-content", url_args={"q": "sha1_git:%s" % target}, request=request + ) + elif target_type == "dir": + directory["target_url"] = reverse( + "api-1-directory", url_args={"sha1_git": target}, request=request + ) else: - directory['target_url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': target}, - request=request) + directory["target_url"] = reverse( + "api-1-revision", url_args={"sha1_git": target}, request=request + ) return directory -def enrich_metadata_endpoint(content_metadata: Dict[str, str], - request: Optional[HttpRequest] = None - ) -> Dict[str, str]: +def enrich_metadata_endpoint( + content_metadata: Dict[str, str], request: Optional[HttpRequest] = None +) -> Dict[str, str]: """Enrich content metadata dict with link to the upper metadata endpoint. Args: content_metadata: dict of data associated to a swh content metadata request: Absolute URIs will be generated if provided Returns: An enriched content metadata dict filled with an additional url """ c = content_metadata - c['content_url'] = reverse('api-1-content', - url_args={'q': 'sha1:%s' % c['id']}, - request=request) + c["content_url"] = reverse( + "api-1-content", url_args={"q": "sha1:%s" % c["id"]}, request=request + ) return c -def enrich_content(content: Dict[str, Any], - top_url: Optional[bool] = False, - query_string: Optional[str] = None, - request: Optional[HttpRequest] = None) -> Dict[str, str]: +def enrich_content( + content: Dict[str, Any], + top_url: Optional[bool] = False, + query_string: Optional[str] = None, + request: Optional[HttpRequest] = None, +) -> Dict[str, str]: """Enrich content with links to: - data_url: its raw data - filetype_url: its filetype information - language_url: its programming language information - license_url: its licensing information Args: content: dict of data associated to a swh content object top_url: whether or not to include the content url in the enriched data query_string: optional query string of type ':' used when requesting the content, it acts as a hint for picking the same hash method when computing the url listed above request: Absolute URIs will be generated if provided Returns: An enriched content dict filled with additional urls """ checksums = content - if 'checksums' in content: - checksums = content['checksums'] - hash_algo = 'sha1' + if "checksums" in content: + checksums = content["checksums"] + hash_algo = "sha1" if query_string: hash_algo = parse_hash(query_string)[0] if hash_algo in checksums: - q = '%s:%s' % (hash_algo, checksums[hash_algo]) + q = "%s:%s" % (hash_algo, checksums[hash_algo]) if top_url: - content['content_url'] = reverse( - 'api-1-content', url_args={'q': q}) - content['data_url'] = reverse('api-1-content-raw', - url_args={'q': q}, - request=request) - content['filetype_url'] = reverse( - 'api-1-content-filetype', - url_args={'q': q}, - request=request) - content['language_url'] = reverse( - 'api-1-content-language', - url_args={'q': q}, - request=request) - content['license_url'] = reverse( - 'api-1-content-license', - url_args={'q': q}, - request=request) + content["content_url"] = reverse("api-1-content", url_args={"q": q}) + content["data_url"] = reverse( + "api-1-content-raw", url_args={"q": q}, request=request + ) + content["filetype_url"] = reverse( + "api-1-content-filetype", url_args={"q": q}, request=request + ) + content["language_url"] = reverse( + "api-1-content-language", url_args={"q": q}, request=request + ) + content["license_url"] = reverse( + "api-1-content-license", url_args={"q": q}, request=request + ) return content -def enrich_revision(revision: Dict[str, Any], - request: Optional[HttpRequest] = None) -> Dict[str, Any]: +def enrich_revision( + revision: Dict[str, Any], request: Optional[HttpRequest] = None +) -> Dict[str, Any]: """Enrich revision with links where it makes sense (directory, parents). Keep track of the navigation breadcrumbs if they are specified. Args: revision: the revision as a dict request: Absolute URIs will be generated if provided Returns: An enriched revision dict filled with additional urls """ - revision['url'] = reverse('api-1-revision', - url_args={'sha1_git': revision['id']}, - request=request) - revision['history_url'] = reverse('api-1-revision-log', - url_args={'sha1_git': revision['id']}, - request=request) - - if 'directory' in revision: - revision['directory_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': revision['directory']}, - request=request) - - if 'parents' in revision: + revision["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision["id"]}, request=request + ) + revision["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision["id"]}, request=request + ) + + if "directory" in revision: + revision["directory_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": revision["directory"]}, + request=request, + ) + + if "parents" in revision: parents = [] - for parent in revision['parents']: - parents.append({ - 'id': parent, - 'url': reverse('api-1-revision', - url_args={'sha1_git': parent}, - request=request) - }) - - revision['parents'] = parents - - if 'children' in revision: + for parent in revision["parents"]: + parents.append( + { + "id": parent, + "url": reverse( + "api-1-revision", url_args={"sha1_git": parent}, request=request + ), + } + ) + + revision["parents"] = parents + + if "children" in revision: children = [] - for child in revision['children']: - children.append(reverse( - 'api-1-revision', - url_args={'sha1_git': child}, - request=request)) - revision['children_urls'] = children - - if 'message_decoding_failed' in revision: - revision['message_url'] = reverse( - 'api-1-revision-raw-message', - url_args={'sha1_git': revision['id']}, - request=request) + for child in revision["children"]: + children.append( + reverse("api-1-revision", url_args={"sha1_git": child}, request=request) + ) + revision["children_urls"] = children + + if "message_decoding_failed" in revision: + revision["message_url"] = reverse( + "api-1-revision-raw-message", + url_args={"sha1_git": revision["id"]}, + request=request, + ) return revision -def enrich_snapshot(snapshot: Dict[str, Any], - request: Optional[HttpRequest] = None) -> Dict[str, Any]: +def enrich_snapshot( + snapshot: Dict[str, Any], request: Optional[HttpRequest] = None +) -> Dict[str, Any]: """Enrich snapshot with links to the branch targets Args: snapshot: the snapshot as a dict request: Absolute URIs will be generated if provided Returns: An enriched snapshot dict filled with additional urls """ - if 'branches' in snapshot: - snapshot['branches'] = { + if "branches" in snapshot: + snapshot["branches"] = { k: enrich_object(v, request) if v else None - for k, v in snapshot['branches'].items() + for k, v in snapshot["branches"].items() } - for k, v in snapshot['branches'].items(): - if v and v['target_type'] == 'alias': + for k, v in snapshot["branches"].items(): + if v and v["target_type"] == "alias": branch = resolve_branch_alias(snapshot, v) if branch: branch = enrich_object(branch, request) - v['target_url'] = branch['target_url'] + v["target_url"] = branch["target_url"] return snapshot -def enrich_origin(origin: Dict[str, Any], - request: Optional[HttpRequest] = None) -> Dict[str, Any]: +def enrich_origin( + origin: Dict[str, Any], request: Optional[HttpRequest] = None +) -> Dict[str, Any]: """Enrich origin dict with link to its visits Args: origin: the origin as a dict request: Absolute URIs will be generated if provided Returns: An enriched origin dict filled with an additional url """ - if 'url' in origin: - origin['origin_visits_url'] = reverse( - 'api-1-origin-visits', - url_args={'origin_url': origin['url']}, - request=request) + if "url" in origin: + origin["origin_visits_url"] = reverse( + "api-1-origin-visits", + url_args={"origin_url": origin["url"]}, + request=request, + ) return origin -def enrich_origin_visit(origin_visit: Dict[str, Any], *, - with_origin_link: bool, with_origin_visit_link: bool, - request: Optional[HttpRequest] = None - ) -> Dict[str, Any]: +def enrich_origin_visit( + origin_visit: Dict[str, Any], + *, + with_origin_link: bool, + with_origin_visit_link: bool, + request: Optional[HttpRequest] = None, +) -> Dict[str, Any]: """Enrich origin visit dict with additional links Args: origin_visit: the origin visit as a dict with_origin_link: whether to add link to origin with_origin_visit_link: whether to add link to origin visit request: Absolute URIs will be generated if provided Returns: An enriched origin visit dict filled with additional urls """ ov = origin_visit if with_origin_link: - ov['origin_url'] = reverse('api-1-origin', - url_args={'origin_url': ov['origin']}, - request=request) + ov["origin_url"] = reverse( + "api-1-origin", url_args={"origin_url": ov["origin"]}, request=request + ) if with_origin_visit_link: - ov['origin_visit_url'] = reverse('api-1-origin-visit', - url_args={'origin_url': ov['origin'], - 'visit_id': ov['visit']}, - request=request) - snapshot = ov['snapshot'] + ov["origin_visit_url"] = reverse( + "api-1-origin-visit", + url_args={"origin_url": ov["origin"], "visit_id": ov["visit"]}, + request=request, + ) + snapshot = ov["snapshot"] if snapshot: - ov['snapshot_url'] = reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}, - request=request) + ov["snapshot_url"] = reverse( + "api-1-snapshot", url_args={"snapshot_id": snapshot}, request=request + ) else: - ov['snapshot_url'] = None + ov["snapshot_url"] = None return ov diff --git a/swh/web/api/views/content.py b/swh/web/api/views/content.py index b98ba4ac..6d26f521 100644 --- a/swh/web/api/views/content.py +++ b/swh/web/api/views/content.py @@ -1,372 +1,409 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from django.http import HttpResponse from swh.web.common import service from swh.web.common.utils import reverse from swh.web.common.exc import NotFoundExc from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api import utils from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/filetype/', - 'api-1-content-filetype', checksum_args=['q']) -@api_doc('/content/filetype/') +@api_route( + r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/filetype/", + "api-1-content-filetype", + checksum_args=["q"], +) +@api_doc("/content/filetype/") @format_docstring() def api_content_filetype(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/filetype/ Get information about the detected MIME type of a content object. - :param string hash_type: optional parameter specifying which hashing algorithm has been used - to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` - or ``blake2s256``. If that parameter is not provided, it is assumed that the - hashing algorithm used is `sha1`. - :param string hash: hexadecimal representation of the checksum value computed with - the specified hashing algorithm. + :param string hash_type: optional parameter specifying which hashing algorithm + has been used to compute the content checksum. It can be either ``sha1``, + ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not + provided, it is assumed that the hashing algorithm used is `sha1`. + :param string hash: hexadecimal representation of the checksum value computed + with the specified hashing algorithm. - :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for - getting information about the content + :>json object content_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information + about the content :>json string encoding: the detected content encoding :>json string id: the **sha1** identifier of the content :>json string mimetype: the detected MIME type of the content - :>json object tool: information about the tool used to detect the content filetype + :>json object tool: information about the tool used to detect the content + filetype {common_headers} :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/filetype/` - """ # noqa + """ return api_lookup( - service.lookup_content_filetype, q, - notfound_msg='No filetype information found for content {}.'.format(q), + service.lookup_content_filetype, + q, + notfound_msg="No filetype information found for content {}.".format(q), enrich_fn=utils.enrich_metadata_endpoint, - request=request) + request=request, + ) -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/language/', - 'api-1-content-language', checksum_args=['q']) -@api_doc('/content/language/') +@api_route( + r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/language/", + "api-1-content-language", + checksum_args=["q"], +) +@api_doc("/content/language/") @format_docstring() def api_content_language(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/language/ Get information about the programming language used in a content object. Note: this endpoint currently returns no data. - :param string hash_type: optional parameter specifying which hashing algorithm has been used - to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` - or ``blake2s256``. If that parameter is not provided, it is assumed that the - hashing algorithm used is ``sha1``. - :param string hash: hexadecimal representation of the checksum value computed with - the specified hashing algorithm. + :param string hash_type: optional parameter specifying which hashing algorithm + has been used to compute the content checksum. It can be either ``sha1``, + ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not + provided, it is assumed that the hashing algorithm used is ``sha1``. + :param string hash: hexadecimal representation of the checksum value computed + with the specified hashing algorithm. - :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for - getting information about the content + :>json object content_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information + about the content :>json string id: the **sha1** identifier of the content :>json string lang: the detected programming language if any - :>json object tool: information about the tool used to detect the programming language + :>json object tool: information about the tool used to detect the programming + language {common_headers} :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/language/` - """ # noqa + """ return api_lookup( - service.lookup_content_language, q, - notfound_msg='No language information found for content {}.'.format(q), + service.lookup_content_language, + q, + notfound_msg="No language information found for content {}.".format(q), enrich_fn=utils.enrich_metadata_endpoint, - request=request) + request=request, + ) -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/license/', - 'api-1-content-license', checksum_args=['q']) -@api_doc('/content/license/') +@api_route( + r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/license/", + "api-1-content-license", + checksum_args=["q"], +) +@api_doc("/content/license/") @format_docstring() def api_content_license(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/license/ Get information about the license of a content object. - :param string hash_type: optional parameter specifying which hashing algorithm has been used - to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` - or ``blake2s256``. If that parameter is not provided, it is assumed that the - hashing algorithm used is ``sha1``. - :param string hash: hexadecimal representation of the checksum value computed with - the specified hashing algorithm. + :param string hash_type: optional parameter specifying which hashing algorithm + has been used to compute the content checksum. It can be either ``sha1``, + ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not + provided, it is assumed that the hashing algorithm used is ``sha1``. + :param string hash: hexadecimal representation of the checksum value computed + with the specified hashing algorithm. - :>json object content_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` for - getting information about the content + :>json object content_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/` for getting information + about the content :>json string id: the **sha1** identifier of the content - :>json array licenses: array of strings containing the detected license names if any + :>json array licenses: array of strings containing the detected license names :>json object tool: information about the tool used to detect the license {common_headers} :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/license/` - """ # noqa + """ return api_lookup( - service.lookup_content_license, q, - notfound_msg='No license information found for content {}.'.format(q), + service.lookup_content_license, + q, + notfound_msg="No license information found for content {}.".format(q), enrich_fn=utils.enrich_metadata_endpoint, - request=request) + request=request, + ) -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/ctags/', - 'api-1-content-ctags') -@api_doc('/content/ctags/', tags=['hidden']) +@api_route(r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/ctags/", "api-1-content-ctags") +@api_doc("/content/ctags/", tags=["hidden"]) def api_content_ctags(request, q): """ Get information about all `Ctags `_-style symbols defined in a content object. """ return api_lookup( - service.lookup_content_ctags, q, - notfound_msg='No ctags symbol found for content {}.'.format(q), + service.lookup_content_ctags, + q, + notfound_msg="No ctags symbol found for content {}.".format(q), enrich_fn=utils.enrich_metadata_endpoint, - request=request) + request=request, + ) -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/raw/', 'api-1-content-raw', - checksum_args=['q']) -@api_doc('/content/raw/', handle_response=True) +@api_route( + r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/raw/", + "api-1-content-raw", + checksum_args=["q"], +) +@api_doc("/content/raw/", handle_response=True) def api_content_raw(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/raw/ Get the raw content of a content object (aka a "blob"), as a byte sequence. - :param string hash_type: optional parameter specifying which hashing algorithm has been used - to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` - or ``blake2s256``. If that parameter is not provided, it is assumed that the - hashing algorithm used is ``sha1``. - :param string hash: hexadecimal representation of the checksum value computed with - the specified hashing algorithm. - :query string filename: if provided, the downloaded content will get that filename + :param string hash_type: optional parameter specifying which hashing algorithm + has been used to compute the content checksum. It can be either ``sha1``, + ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not + provided, it is assumed that the hashing algorithm used is ``sha1``. + :param string hash: hexadecimal representation of the checksum value computed + with the specified hashing algorithm. + :query string filename: if provided, the downloaded content will get that + filename :resheader Content-Type: application/octet-stream :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`content/sha1:dc2830a9e72f23c1dfebef4413003221baa5fb62/raw/` - """ # noqa + """ + def generate(content): - yield content['data'] + yield content["data"] content_raw = service.lookup_content_raw(q) if not content_raw: - raise NotFoundExc('Content %s is not found.' % q) + raise NotFoundExc("Content %s is not found." % q) - filename = request.query_params.get('filename') + filename = request.query_params.get("filename") if not filename: - filename = 'content_%s_raw' % q.replace(':', '_') + filename = "content_%s_raw" % q.replace(":", "_") - response = HttpResponse(generate(content_raw), - content_type='application/octet-stream') - response['Content-disposition'] = 'attachment; filename=%s' % filename + response = HttpResponse( + generate(content_raw), content_type="application/octet-stream" + ) + response["Content-disposition"] = "attachment; filename=%s" % filename return response -@api_route(r'/content/symbol/(?P.+)/', 'api-1-content-symbol') -@api_doc('/content/symbol/', tags=['hidden']) +@api_route(r"/content/symbol/(?P.+)/", "api-1-content-symbol") +@api_doc("/content/symbol/", tags=["hidden"]) def api_content_symbol(request, q=None): """Search content objects by `Ctags `_-style symbol (e.g., function name, data type, method, ...). """ result = {} - last_sha1 = request.query_params.get('last_sha1', None) - per_page = int(request.query_params.get('per_page', '10')) + last_sha1 = request.query_params.get("last_sha1", None) + per_page = int(request.query_params.get("per_page", "10")) def lookup_exp(exp, last_sha1=last_sha1, per_page=per_page): exp = list(service.lookup_expression(exp, last_sha1, per_page)) return exp if exp else None symbols = api_lookup( - lookup_exp, q, + lookup_exp, + q, notfound_msg="No indexed raw content match expression '{}'.".format(q), enrich_fn=functools.partial(utils.enrich_content, top_url=True), - request=request) + request=request, + ) if symbols: nb_symbols = len(symbols) if nb_symbols == per_page: query_params = {} - new_last_sha1 = symbols[-1]['sha1'] - query_params['last_sha1'] = new_last_sha1 - if request.query_params.get('per_page'): - query_params['per_page'] = per_page - - result['headers'] = { - 'link-next': reverse('api-1-content-symbol', url_args={'q': q}, - query_params=query_params, - request=request) + new_last_sha1 = symbols[-1]["sha1"] + query_params["last_sha1"] = new_last_sha1 + if request.query_params.get("per_page"): + query_params["per_page"] = per_page + + result["headers"] = { + "link-next": reverse( + "api-1-content-symbol", + url_args={"q": q}, + query_params=query_params, + request=request, + ) } - result.update({ - 'results': symbols - }) + result.update({"results": symbols}) return result -@api_route(r'/content/known/search/', 'api-1-content-known', methods=['POST']) -@api_route(r'/content/known/(?P(?!search).*)/', 'api-1-content-known') -@api_doc('/content/known/', tags=['hidden']) +@api_route(r"/content/known/search/", "api-1-content-known", methods=["POST"]) +@api_route(r"/content/known/(?P(?!search).*)/", "api-1-content-known") +@api_doc("/content/known/", tags=["hidden"]) @format_docstring() def api_check_content_known(request, q=None): """ .. http:get:: /api/1/content/known/(sha1)[,(sha1), ...,(sha1)]/ Check whether some content(s) (aka "blob(s)") is present in the archive based on its **sha1** checksum. :param string sha1: hexadecimal representation of the **sha1** checksum value - for the content to check existence. Multiple values can be provided separated - by ','. + for the content to check existence. Multiple values can be provided + separated by ','. {common_headers} - :>json array search_res: array holding the search result for each provided **sha1** - :>json object search_stats: some statistics regarding the number of **sha1** provided - and the percentage of those found in the archive + :>json array search_res: array holding the search result for each provided + **sha1** + :>json object search_stats: some statistics regarding the number of **sha1** + provided and the percentage of those found in the archive :statuscode 200: no error :statuscode 400: an invalid **sha1** has been provided **Example:** .. parsed-literal:: :swh_web_api:`content/known/dc2830a9e72f23c1dfebef4413003221baa5fb62,0c3f19cb47ebfbe643fb19fa94c874d18fa62d12/` - """ # noqa - response = {'search_res': None, - 'search_stats': None} - search_stats = {'nbfiles': 0, 'pct': 0} + """ + response = {"search_res": None, "search_stats": None} + search_stats = {"nbfiles": 0, "pct": 0} search_res = None queries = [] # GET: Many hash separated values request if q: - hashes = q.split(',') + hashes = q.split(",") for v in hashes: - queries.append({'filename': None, 'sha1': v}) + queries.append({"filename": None, "sha1": v}) # POST: Many hash requests in post form submission - elif request.method == 'POST': + elif request.method == "POST": data = request.data # Remove potential inputs with no associated value for k, v in data.items(): if v is not None: - if k == 'q' and len(v) > 0: - queries.append({'filename': None, 'sha1': v}) - elif v != '': - queries.append({'filename': k, 'sha1': v}) + if k == "q" and len(v) > 0: + queries.append({"filename": None, "sha1": v}) + elif v != "": + queries.append({"filename": k, "sha1": v}) if queries: lookup = service.lookup_multiple_hashes(queries) result = [] nb_queries = len(queries) for el in lookup: - res_d = {'sha1': el['sha1'], - 'found': el['found']} - if 'filename' in el and el['filename']: - res_d['filename'] = el['filename'] + res_d = {"sha1": el["sha1"], "found": el["found"]} + if "filename" in el and el["filename"]: + res_d["filename"] = el["filename"] result.append(res_d) search_res = result - nbfound = len([x for x in lookup if x['found']]) - search_stats['nbfiles'] = nb_queries - search_stats['pct'] = (nbfound / nb_queries) * 100 + nbfound = len([x for x in lookup if x["found"]]) + search_stats["nbfiles"] = nb_queries + search_stats["pct"] = (nbfound / nb_queries) * 100 - response['search_res'] = search_res - response['search_stats'] = search_stats + response["search_res"] = search_res + response["search_stats"] = search_stats return response -@api_route(r'/content/(?P[0-9a-z_:]*[0-9a-f]+)/', 'api-1-content', - checksum_args=['q']) -@api_doc('/content/') +@api_route( + r"/content/(?P[0-9a-z_:]*[0-9a-f]+)/", "api-1-content", checksum_args=["q"] +) +@api_doc("/content/") @format_docstring() def api_content_metadata(request, q): """ .. http:get:: /api/1/content/[(hash_type):](hash)/ Get information about a content (aka a "blob") object. In the archive, a content object is identified based on checksum values computed using various hashing algorithms. - :param string hash_type: optional parameter specifying which hashing algorithm has been used - to compute the content checksum. It can be either ``sha1``, ``sha1_git``, ``sha256`` - or ``blake2s256``. If that parameter is not provided, it is assumed that the - hashing algorithm used is ``sha1``. - :param string hash: hexadecimal representation of the checksum value computed with - the specified hashing algorithm. + :param string hash_type: optional parameter specifying which hashing algorithm + has been used to compute the content checksum. It can be either ``sha1``, + ``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not + provided, it is assumed that the hashing algorithm used is ``sha1``. + :param string hash: hexadecimal representation of the checksum value computed + with the specified hashing algorithm. {common_headers} - :>json object checksums: object holding the computed checksum values for the requested content - :>json string data_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/raw/` + :>json object checksums: object holding the computed checksum values for the + requested content + :>json string data_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/raw/` for downloading the content raw bytes - :>json string filetype_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/filetype/` + :>json string filetype_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/filetype/` for getting information about the content MIME type - :>json string language_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/language/` + :>json string language_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/language/` for getting information about the programming language used in the content :>json number length: length of the content in bytes - :>json string license_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/license/` + :>json string license_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/license/` for getting information about the license of the content :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested content can not be found in the archive **Example:** .. parsed-literal:: - curl -i :swh_web_api:`content/sha1_git:fe95a46679d128ff167b7c55df5d02356c5a1ae1/` - """ # noqa + :swh_web_api:`content/sha1_git:fe95a46679d128ff167b7c55df5d02356c5a1ae1/` + """ return api_lookup( - service.lookup_content, q, - notfound_msg='Content with {} not found.'.format(q), + service.lookup_content, + q, + notfound_msg="Content with {} not found.".format(q), enrich_fn=functools.partial(utils.enrich_content, query_string=q), - request=request) + request=request, + ) diff --git a/swh/web/api/views/directory.py b/swh/web/api/views/directory.py index 01fec6b2..ab205d2a 100644 --- a/swh/web/api/views/directory.py +++ b/swh/web/api/views/directory.py @@ -1,76 +1,93 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -@api_route(r'/directory/(?P[0-9a-f]+)/', 'api-1-directory', - checksum_args=['sha1_git']) -@api_route(r'/directory/(?P[0-9a-f]+)/(?P.+)/', - 'api-1-directory', - checksum_args=['sha1_git']) -@api_doc('/directory/') +@api_route( + r"/directory/(?P[0-9a-f]+)/", + "api-1-directory", + checksum_args=["sha1_git"], +) +@api_route( + r"/directory/(?P[0-9a-f]+)/(?P.+)/", + "api-1-directory", + checksum_args=["sha1_git"], +) +@api_doc("/directory/") @format_docstring() def api_directory(request, sha1_git, path=None): """ .. http:get:: /api/1/directory/(sha1_git)/[(path)/] Get information about directory objects. - Directories are identified by **sha1** checksums, compatible with Git directory identifiers. - See :func:`swh.model.identifiers.directory_identifier` in our data model module for details - about how they are computed. + Directories are identified by **sha1** checksums, compatible with Git + directory identifiers. + See :func:`swh.model.identifiers.directory_identifier` in our data model + module for details about how they are computed. - When given only a directory identifier, this endpoint returns information about the directory itself, - returning its content (usually a list of directory entries). When given a directory identifier and a - path, this endpoint returns information about the directory entry pointed by the relative path, - starting path resolution from the given directory. + When given only a directory identifier, this endpoint returns information + about the directory itself, returning its content (usually a list of directory + entries). When given a directory identifier and a path, this endpoint returns + information about the directory entry pointed by the relative path, starting + path resolution from the given directory. - :param string sha1_git: hexadecimal representation of the directory **sha1_git** identifier - :param string path: optional parameter to get information about the directory entry - pointed by that relative path + :param string sha1_git: hexadecimal representation of the directory + **sha1_git** identifier + :param string path: optional parameter to get information about the + directory entry pointed by that relative path {common_headers} - :>jsonarr object checksums: object holding the computed checksum values for a directory entry - (only for file entries) + :>jsonarr object checksums: object holding the computed checksum values for + a directory entry (only for file entries) :>jsonarr string dir_id: **sha1_git** identifier of the requested directory - :>jsonarr number length: length of a directory entry in bytes (only for file entries) - for getting information about the content MIME type + :>jsonarr number length: length of a directory entry in bytes (only for file + entries) for getting information about the content MIME type :>jsonarr string name: the directory entry name :>jsonarr number perms: permissions for the directory entry :>jsonarr string target: **sha1_git** identifier of the directory entry - :>jsonarr string target_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/` - or :http:get:`/api/1/directory/(sha1_git)/[(path)/]` depending on the directory entry type - :>jsonarr string type: the type of the directory entry, can be either ``dir``, ``file`` or ``rev`` + :>jsonarr string target_url: link to + :http:get:`/api/1/content/[(hash_type):](hash)/` + or :http:get:`/api/1/directory/(sha1_git)/[(path)/]` + depending on the directory entry type + :>jsonarr string type: the type of the directory entry, can be either + ``dir``, ``file`` or ``rev`` :statuscode 200: no error :statuscode 400: an invalid **hash_type** or **hash** has been provided :statuscode 404: requested directory can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`directory/977fc4b98c0e85816348cebd3b12026407c368b6/` - """ # noqa + """ if path: - error_msg_path = ('Entry with path %s relative to directory ' - 'with sha1_git %s not found.') % (path, sha1_git) + error_msg_path = ( + "Entry with path %s relative to directory " "with sha1_git %s not found." + ) % (path, sha1_git) return api_lookup( - service.lookup_directory_with_path, sha1_git, path, + service.lookup_directory_with_path, + sha1_git, + path, notfound_msg=error_msg_path, enrich_fn=utils.enrich_directory, - request=request) + request=request, + ) else: - error_msg_nopath = 'Directory with sha1_git %s not found.' % sha1_git + error_msg_nopath = "Directory with sha1_git %s not found." % sha1_git return api_lookup( - service.lookup_directory, sha1_git, + service.lookup_directory, + sha1_git, notfound_msg=error_msg_nopath, enrich_fn=utils.enrich_directory, - request=request) + request=request, + ) diff --git a/swh/web/api/views/identifiers.py b/swh/web/api/views/identifiers.py index 4b0da537..4fda5fe0 100644 --- a/swh/web/api/views/identifiers.py +++ b/swh/web/api/views/identifiers.py @@ -1,120 +1,120 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common import service from swh.web.common.exc import LargePayloadExc from swh.web.common.identifiers import ( - resolve_swh_persistent_id, get_persistent_identifier, - group_swh_persistent_identifiers + resolve_swh_persistent_id, + get_persistent_identifier, + group_swh_persistent_identifiers, ) -@api_route(r'/resolve/(?P.*)/', - 'api-1-resolve-swh-pid') -@api_doc('/resolve/') +@api_route(r"/resolve/(?P.*)/", "api-1-resolve-swh-pid") +@api_doc("/resolve/") @format_docstring() def api_resolve_swh_pid(request, swh_id): """ .. http:get:: /api/1/resolve/(swh_id)/ Resolve a Software Heritage persistent identifier. Try to resolve a provided `persistent identifier `_ into an url for browsing the pointed archive object. If the provided identifier is valid, the existence of the object in the archive will also be checked. :param string swh_id: a Software Heritage persistent identifier :>json string browse_url: the url for browsing the pointed object :>json object metadata: object holding optional parts of the persistent identifier :>json string namespace: the persistent identifier namespace :>json string object_id: the hash identifier of the pointed object :>json string object_type: the type of the pointed object :>json number scheme_version: the scheme version of the persistent identifier {common_headers} :statuscode 200: no error :statuscode 400: an invalid persistent identifier has been provided :statuscode 404: the pointed object does not exist in the archive **Example:** .. parsed-literal:: :swh_web_api:`resolve/swh:1:rev:96db9023b881d7cd9f379b0c154650d6c108e9a3;origin=https://github.com/openssl/openssl/` """ # try to resolve the provided pid swh_id_resolved = resolve_swh_persistent_id(swh_id) # id is well-formed, now check that the pointed # object is present in the archive, NotFoundExc # will be raised otherwise - swh_id_parsed = swh_id_resolved['swh_id_parsed'] + swh_id_parsed = swh_id_resolved["swh_id_parsed"] object_type = swh_id_parsed.object_type object_id = swh_id_parsed.object_id service.lookup_object(object_type, object_id) # id is well-formed and the pointed object exists swh_id_data = swh_id_parsed._asdict() - swh_id_data['browse_url'] = request.build_absolute_uri( - swh_id_resolved['browse_url']) + swh_id_data["browse_url"] = request.build_absolute_uri( + swh_id_resolved["browse_url"] + ) return swh_id_data -@api_route(r'/known/', - 'api-1-known', methods=['POST']) -@api_doc('/known/') +@api_route(r"/known/", "api-1-known", methods=["POST"]) +@api_doc("/known/") @format_docstring() def api_swh_pid_known(request): """ .. http:post:: /api/1/known/ Check if a list of objects are present in the Software Heritage archive. The objects to check existence must be provided using Software Heritage `persistent identifiers `_. :json object : an object whose keys are input persistent identifiers and values objects with the following keys: * **known (bool)**: whether the object was found {common_headers} :statuscode 200: no error :statuscode 400: an invalid persistent identifier was provided :statuscode 413: the input array of persistent identifiers is too large """ limit = 1000 if len(request.data) > limit: - raise LargePayloadExc('The maximum number of PIDs this endpoint can ' - 'receive is %s' % limit) + raise LargePayloadExc( + "The maximum number of PIDs this endpoint can " "receive is %s" % limit + ) - persistent_ids = [get_persistent_identifier(pid) - for pid in request.data] + persistent_ids = [get_persistent_identifier(pid) for pid in request.data] - response = {str(pid): {'known': False} for pid in persistent_ids} + response = {str(pid): {"known": False} for pid in persistent_ids} # group pids by their type pids_by_type = group_swh_persistent_identifiers(persistent_ids) # search for hashes not present in the storage missing_hashes = service.lookup_missing_hashes(pids_by_type) for pid in persistent_ids: if pid.object_id not in missing_hashes: - response[str(pid)]['known'] = True + response[str(pid)]["known"] = True return response diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py index c6729eb4..4d5afab3 100644 --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -1,459 +1,475 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.util import strtobool from functools import partial from swh.web.common import service from swh.web.common.exc import BadInputExc from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import reverse from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.utils import enrich_origin, enrich_origin_visit from swh.web.api.views.utils import api_lookup -DOC_RETURN_ORIGIN = ''' +DOC_RETURN_ORIGIN = """ :>json string origin_visits_url: link to in order to get information about the visits for that origin :>json string url: the origin canonical url -''' +""" -DOC_RETURN_ORIGIN_ARRAY = \ - DOC_RETURN_ORIGIN.replace(':>json', ':>jsonarr') +DOC_RETURN_ORIGIN_ARRAY = DOC_RETURN_ORIGIN.replace(":>json", ":>jsonarr") -DOC_RETURN_ORIGIN_VISIT = ''' +DOC_RETURN_ORIGIN_VISIT = """ :>json string date: ISO representation of the visit date (in UTC) :>json str origin: the origin canonical url :>json string origin_url: link to get information about the origin :>jsonarr string snapshot: the snapshot identifier of the visit (may be null if status is not **full**). :>jsonarr string snapshot_url: link to :http:get:`/api/1/snapshot/(snapshot_id)/` in order to get information about the snapshot of the visit (may be null if status is not **full**). :>json string status: status of the visit (either **full**, **partial** or **ongoing**) :>json number visit: the unique identifier of the visit -''' +""" -DOC_RETURN_ORIGIN_VISIT_ARRAY = \ - DOC_RETURN_ORIGIN_VISIT.replace(':>json', ':>jsonarr') +DOC_RETURN_ORIGIN_VISIT_ARRAY = DOC_RETURN_ORIGIN_VISIT.replace(":>json", ":>jsonarr") -DOC_RETURN_ORIGIN_VISIT_ARRAY += ''' +DOC_RETURN_ORIGIN_VISIT_ARRAY += """ :>jsonarr number id: the unique identifier of the origin :>jsonarr string origin_visit_url: link to :http:get:`/api/1/origin/(origin_url)/visit/(visit_id)/` in order to get information about the visit -''' +""" -@api_route(r'/origins/', 'api-1-origins') -@api_doc('/origins/', noargs=True) +@api_route(r"/origins/", "api-1-origins") +@api_doc("/origins/", noargs=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origins(request): """ .. http:get:: /api/1/origins/ Get list of archived software origins. .. warning:: This endpoint used to provide an `origin_from` query parameter, and guarantee an order on results. This is no longer true, and only the Link header should be used for paginating through results. :query int origin_count: The maximum number of origins to return (default to 100, can not exceed 10000) {return_origin_array} {common_headers} {resheader_link} :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origins?origin_count=500` """ - origin_from = int(request.query_params.get('origin_from', '1')) - origin_count = int(request.query_params.get('origin_count', '100')) + origin_from = int(request.query_params.get("origin_from", "1")) + origin_count = int(request.query_params.get("origin_count", "100")) origin_count = min(origin_count, 10000) results = api_lookup( - service.lookup_origins, origin_from, origin_count+1, + service.lookup_origins, + origin_from, + origin_count + 1, enrich_fn=enrich_origin, - request=request) - response = {'results': results, 'headers': {}} + request=request, + ) + response = {"results": results, "headers": {}} if len(results) > origin_count: - origin_from = results.pop()['id'] - response['headers']['link-next'] = reverse( - 'api-1-origins', - query_params={'origin_from': origin_from, - 'origin_count': origin_count}, - request=request) + origin_from = results.pop()["id"] + response["headers"]["link-next"] = reverse( + "api-1-origins", + query_params={"origin_from": origin_from, "origin_count": origin_count}, + request=request, + ) for result in results: - if 'id' in result: - del result['id'] + if "id" in result: + del result["id"] return response -@api_route(r'/origin/(?P.+)/get/', 'api-1-origin') -@api_doc('/origin/') +@api_route(r"/origin/(?P.+)/get/", "api-1-origin") +@api_doc("/origin/") @format_docstring(return_origin=DOC_RETURN_ORIGIN) def api_origin(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/get/ Get information about a software origin. :param string origin_url: the origin url {return_origin} {common_headers} :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/python/cpython/get/` """ - ori_dict = { - 'url': origin_url - } + ori_dict = {"url": origin_url} - error_msg = 'Origin with url %s not found.' % ori_dict['url'] + error_msg = "Origin with url %s not found." % ori_dict["url"] return api_lookup( - service.lookup_origin, ori_dict, + service.lookup_origin, + ori_dict, notfound_msg=error_msg, enrich_fn=enrich_origin, - request=request) + request=request, + ) -@api_route(r'/origin/search/(?P.+)/', - 'api-1-origin-search', - throttle_scope='swh_api_origin_search') -@api_doc('/origin/search/') +@api_route( + r"/origin/search/(?P.+)/", + "api-1-origin-search", + throttle_scope="swh_api_origin_search", +) +@api_doc("/origin/search/") @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origin_search(request, url_pattern): """ .. http:get:: /api/1/origin/search/(url_pattern)/ Search for software origins whose urls contain a provided string pattern or match a provided regular expression. The search is performed in a case insensitive way. .. warning:: This endpoint used to provide an `offset` query parameter, and guarantee an order on results. This is no longer true, and only the Link header should be used for paginating through results. :param string url_pattern: a string pattern :query int limit: the maximum number of found origins to return (bounded to 1000) :query boolean with_visit: if true, only return origins with at least one visit by Software heritage {return_origin_array} {common_headers} {resheader_link} :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/search/python/?limit=2` """ result = {} - limit = min(int(request.query_params.get('limit', '70')), 1000) - page_token = request.query_params.get('page_token') - with_visit = request.query_params.get('with_visit', 'false') + limit = min(int(request.query_params.get("limit", "70")), 1000) + page_token = request.query_params.get("page_token") + with_visit = request.query_params.get("with_visit", "false") (results, page_token) = api_lookup( - service.search_origin, url_pattern, limit, - bool(strtobool(with_visit)), page_token, - enrich_fn=enrich_origin, request=request) + service.search_origin, + url_pattern, + limit, + bool(strtobool(with_visit)), + page_token, + enrich_fn=enrich_origin, + request=request, + ) if page_token is not None: query_params = {} - query_params['limit'] = limit - query_params['page_token'] = page_token - - result['headers'] = { - 'link-next': reverse('api-1-origin-search', - url_args={'url_pattern': url_pattern}, - query_params=query_params, - request=request) + query_params["limit"] = limit + query_params["page_token"] = page_token + + result["headers"] = { + "link-next": reverse( + "api-1-origin-search", + url_args={"url_pattern": url_pattern}, + query_params=query_params, + request=request, + ) } - result.update({ - 'results': results - }) + result.update({"results": results}) return result -@api_route(r'/origin/metadata-search/', - 'api-1-origin-metadata-search') -@api_doc('/origin/metadata-search/', noargs=True, need_params=True) +@api_route(r"/origin/metadata-search/", "api-1-origin-metadata-search") +@api_doc("/origin/metadata-search/", noargs=True, need_params=True) @format_docstring(return_origin_array=DOC_RETURN_ORIGIN_ARRAY) def api_origin_metadata_search(request): """ .. http:get:: /api/1/origin/metadata-search/ Search for software origins whose metadata (expressed as a JSON-LD/CodeMeta dictionary) match the provided criteria. For now, only full-text search on this dictionary is supported. :query str fulltext: a string that will be matched against origin metadata; results are ranked and ordered starting with the best ones. :query int limit: the maximum number of found origins to return (bounded to 100) {return_origin_array} {common_headers} :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`origin/metadata-search/?limit=2&fulltext=Jane%20Doe` """ - fulltext = request.query_params.get('fulltext', None) - limit = min(int(request.query_params.get('limit', '70')), 100) + fulltext = request.query_params.get("fulltext", None) + limit = min(int(request.query_params.get("limit", "70")), 100) if not fulltext: content = '"fulltext" must be provided and non-empty.' raise BadInputExc(content) - results = api_lookup(service.search_origin_metadata, fulltext, limit, - request=request) + results = api_lookup( + service.search_origin_metadata, fulltext, limit, request=request + ) return { - 'results': results, + "results": results, } -@api_route(r'/origin/(?P.*)/visits/', 'api-1-origin-visits') -@api_doc('/origin/visits/') -@format_docstring( - return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY) +@api_route(r"/origin/(?P.*)/visits/", "api-1-origin-visits") +@api_doc("/origin/visits/") +@format_docstring(return_origin_visit_array=DOC_RETURN_ORIGIN_VISIT_ARRAY) def api_origin_visits(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visits/ Get information about all visits of a software origin. Visits are returned sorted in descending order according to their date. :param str origin_url: a software origin URL :query int per_page: specify the number of visits to list, for pagination purposes :query int last_visit: visit to start listing from, for pagination purposes {common_headers} {resheader_link} {return_origin_visit_array} :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visits/` """ result = {} - origin_query = {'url': origin_url} - notfound_msg = 'No origin {} found'.format(origin_url) - url_args_next = {'origin_url': origin_url} - per_page = int(request.query_params.get('per_page', '10')) - last_visit = request.query_params.get('last_visit') + origin_query = {"url": origin_url} + notfound_msg = "No origin {} found".format(origin_url) + url_args_next = {"origin_url": origin_url} + per_page = int(request.query_params.get("per_page", "10")) + last_visit = request.query_params.get("last_visit") if last_visit: last_visit = int(last_visit) - def _lookup_origin_visits( - origin_query, last_visit=last_visit, per_page=per_page): + def _lookup_origin_visits(origin_query, last_visit=last_visit, per_page=per_page): all_visits = get_origin_visits(origin_query) all_visits.reverse() visits = [] if not last_visit: visits = all_visits[:per_page] else: for i, v in enumerate(all_visits): - if v['visit'] == last_visit: - visits = all_visits[i+1:i+1+per_page] + if v["visit"] == last_visit: + visits = all_visits[i + 1 : i + 1 + per_page] break for v in visits: yield v - results = api_lookup(_lookup_origin_visits, origin_query, - notfound_msg=notfound_msg, - enrich_fn=partial(enrich_origin_visit, - with_origin_link=False, - with_origin_visit_link=True), - request=request) + results = api_lookup( + _lookup_origin_visits, + origin_query, + notfound_msg=notfound_msg, + enrich_fn=partial( + enrich_origin_visit, with_origin_link=False, with_origin_visit_link=True + ), + request=request, + ) if results: nb_results = len(results) if nb_results == per_page: - new_last_visit = results[-1]['visit'] + new_last_visit = results[-1]["visit"] query_params = {} - query_params['last_visit'] = new_last_visit - - if request.query_params.get('per_page'): - query_params['per_page'] = per_page - - result['headers'] = { - 'link-next': reverse('api-1-origin-visits', - url_args=url_args_next, - query_params=query_params, - request=request) + query_params["last_visit"] = new_last_visit + + if request.query_params.get("per_page"): + query_params["per_page"] = per_page + + result["headers"] = { + "link-next": reverse( + "api-1-origin-visits", + url_args=url_args_next, + query_params=query_params, + request=request, + ) } - result.update({ - 'results': results - }) + result.update({"results": results}) return result -@api_route(r'/origin/(?P.*)/visit/latest/', - 'api-1-origin-visit-latest', - throttle_scope='swh_api_origin_visit_latest') -@api_doc('/origin/visit/latest/') +@api_route( + r"/origin/(?P.*)/visit/latest/", + "api-1-origin-visit-latest", + throttle_scope="swh_api_origin_visit_latest", +) +@api_doc("/origin/visit/latest/") @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) def api_origin_visit_latest(request, origin_url=None): """ .. http:get:: /api/1/origin/(origin_url)/visit/latest/ Get information about the latest visit of a software origin. :param str origin_url: a software origin URL :query boolean require_snapshot: if true, only return a visit with a snapshot {common_headers} {return_origin_visit} :statuscode 200: no error :statuscode 404: requested origin or visit can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/` """ - require_snapshot = request.query_params.get('require_snapshot', 'false') + require_snapshot = request.query_params.get("require_snapshot", "false") return api_lookup( - service.lookup_origin_visit_latest, origin_url, + service.lookup_origin_visit_latest, + origin_url, bool(strtobool(require_snapshot)), - notfound_msg=('No visit for origin {} found' - .format(origin_url)), - enrich_fn=partial(enrich_origin_visit, - with_origin_link=True, - with_origin_visit_link=False), - request=request) - - -@api_route(r'/origin/(?P.*)/visit/(?P[0-9]+)/', - 'api-1-origin-visit') -@api_doc('/origin/visit/') + notfound_msg=("No visit for origin {} found".format(origin_url)), + enrich_fn=partial( + enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False + ), + request=request, + ) + + +@api_route( + r"/origin/(?P.*)/visit/(?P[0-9]+)/", "api-1-origin-visit" +) +@api_doc("/origin/visit/") @format_docstring(return_origin_visit=DOC_RETURN_ORIGIN_VISIT) def api_origin_visit(request, visit_id, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/ Get information about a specific visit of a software origin. :param str origin_url: a software origin URL :param int visit_id: a visit identifier {common_headers} {return_origin_visit} :statuscode 200: no error :statuscode 404: requested origin or visit can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/hylang/hy/visit/1/` """ return api_lookup( - service.lookup_origin_visit, origin_url, int(visit_id), - notfound_msg=('No visit {} for origin {} found' - .format(visit_id, origin_url)), - enrich_fn=partial(enrich_origin_visit, - with_origin_link=True, - with_origin_visit_link=False), - request=request) - - -@api_route(r'/origin/(?P.+)' - '/intrinsic-metadata', 'api-origin-intrinsic-metadata') -@api_doc('/origin/intrinsic-metadata/') + service.lookup_origin_visit, + origin_url, + int(visit_id), + notfound_msg=("No visit {} for origin {} found".format(visit_id, origin_url)), + enrich_fn=partial( + enrich_origin_visit, with_origin_link=True, with_origin_visit_link=False + ), + request=request, + ) + + +@api_route( + r"/origin/(?P.+)" "/intrinsic-metadata", "api-origin-intrinsic-metadata" +) +@api_doc("/origin/intrinsic-metadata/") @format_docstring() def api_origin_intrinsic_metadata(request, origin_url): """ .. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary). :param string origin_url: the origin url :>json string ???: intrinsic metadata field of the origin {common_headers} :statuscode 200: no error :statuscode 404: requested origin can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`origin/https://github.com/python/cpython/intrinsic-metadata` - """ # noqa - ori_dict = { - 'url': origin_url - } + """ + ori_dict = {"url": origin_url} - error_msg = 'Origin with url %s not found' % ori_dict['url'] + error_msg = "Origin with url %s not found" % ori_dict["url"] return api_lookup( - service.lookup_origin_intrinsic_metadata, ori_dict, + service.lookup_origin_intrinsic_metadata, + ori_dict, notfound_msg=error_msg, enrich_fn=enrich_origin, - request=request) + request=request, + ) diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py index 0696169a..a6b64c11 100644 --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -1,85 +1,89 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.views.decorators.cache import never_cache from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common.origin_save import ( - create_save_origin_request, get_save_origin_requests + create_save_origin_request, + get_save_origin_requests, ) -@api_route(r'/origin/save/(?P.+)/url/(?P.+)/', - 'api-1-save-origin', methods=['GET', 'POST'], - throttle_scope='swh_save_origin') +@api_route( + r"/origin/save/(?P.+)/url/(?P.+)/", + "api-1-save-origin", + methods=["GET", "POST"], + throttle_scope="swh_save_origin", +) @never_cache -@api_doc('/origin/save/') +@api_doc("/origin/save/") @format_docstring() def api_save_origin(request, visit_type, origin_url): """ .. http:get:: /api/1/origin/save/(visit_type)/url/(origin_url)/ .. http:post:: /api/1/origin/save/(visit_type)/url/(origin_url)/ Request the saving of a software origin into the archive or check the status of previously created save requests. That endpoint enables to create a saving task for a software origin through a POST request. Depending of the provided origin url, the save request can either be: * immediately **accepted**, for well known code hosting providers like for instance GitHub or GitLab * **rejected**, in case the url is blacklisted by Software Heritage * **put in pending state** until a manual check is done in order to determine if it can be loaded or not Once a saving request has been accepted, its associated saving task status can then be checked through a GET request on the same url. Returned status can either be: * **not created**: no saving task has been created * **not yet scheduled**: saving task has been created but its execution has not yet been scheduled * **scheduled**: the task execution has been scheduled * **succeed**: the saving task has been successfully executed * **failed**: the saving task has been executed but it failed When issuing a POST request an object will be returned while a GET request will return an array of objects (as multiple save requests might have been submitted for the same origin). :param string visit_type: the type of visit to perform (currently the supported types are ``git``, ``hg`` and ``svn``) :param string origin_url: the url of the origin to save {common_headers} :>json string origin_url: the url of the origin to save :>json string visit_type: the type of visit to perform :>json string save_request_date: the date (in iso format) the save request was issued :>json string save_request_status: the status of the save request, either **accepted**, **rejected** or **pending** :>json string save_task_status: the status of the origin saving task, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** :statuscode 200: no error :statuscode 400: an invalid visit type or origin url has been provided :statuscode 403: the provided origin url is blacklisted :statuscode 404: no save requests have been found for a given origin """ - if request.method == 'POST': + if request.method == "POST": sor = create_save_origin_request(visit_type, origin_url) - del sor['id'] + del sor["id"] else: sor = get_save_origin_requests(visit_type, origin_url) for s in sor: - del s['id'] + del s["id"] return sor diff --git a/swh/web/api/views/ping.py b/swh/web/api/views/ping.py index 9a2a722b..ce97b522 100644 --- a/swh/web/api/views/ping.py +++ b/swh/web/api/views/ping.py @@ -1,21 +1,21 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route -@api_route(r'/ping/', 'api-1-ping') -@api_doc('/ping/', noargs=True) +@api_route(r"/ping/", "api-1-ping") +@api_doc("/ping/", noargs=True) def ping(request): """ .. http:get:: /api/1/ping/ A simple endpoint used to check if the API is working. :statuscode 200: no error """ - return 'pong' + return "pong" diff --git a/swh/web/api/views/release.py b/swh/web/api/views/release.py index f4e827c7..cdf8d98b 100644 --- a/swh/web/api/views/release.py +++ b/swh/web/api/views/release.py @@ -1,57 +1,60 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -@api_route(r'/release/(?P[0-9a-f]+)/', 'api-1-release', - checksum_args=['sha1_git']) -@api_doc('/release/') +@api_route( + r"/release/(?P[0-9a-f]+)/", "api-1-release", checksum_args=["sha1_git"] +) +@api_doc("/release/") @format_docstring() def api_release(request, sha1_git): """ .. http:get:: /api/1/release/(sha1_git)/ Get information about a release in the archive. Releases are identified by **sha1** checksums, compatible with Git tag identifiers. See :func:`swh.model.identifiers.release_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the release **sha1_git** identifier {common_headers} :>json object author: information about the author of the release :>json string date: ISO representation of the release date (in UTC) :>json string id: the release unique identifier :>json string message: the message associated to the release :>json string name: the name of the release :>json string target: the target identifier of the release :>json string target_type: the type of the target, can be either **release**, **revision**, **content**, **directory** :>json string target_url: a link to the adequate api url based on the target type :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested release can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`release/208f61cc7a5dbc9879ae6e5c2f95891e270f09ef/` """ - error_msg = 'Release with sha1_git %s not found.' % sha1_git + error_msg = "Release with sha1_git %s not found." % sha1_git return api_lookup( - service.lookup_release, sha1_git, + service.lookup_release, + sha1_git, notfound_msg=error_msg, enrich_fn=utils.enrich_release, - request=request) + request=request, + ) diff --git a/swh/web/api/views/revision.py b/swh/web/api/views/revision.py index b1047e9a..85e8ef2b 100644 --- a/swh/web/api/views/revision.py +++ b/swh/web/api/views/revision.py @@ -1,251 +1,275 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpResponse from swh.web.common import service from swh.web.common.utils import reverse from swh.web.api import utils from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -DOC_RETURN_REVISION = ''' +DOC_RETURN_REVISION = """ :>json object author: information about the author of the revision :>json object committer: information about the committer of the revision :>json string committer_date: ISO representation of the commit date (in UTC) :>json string date: ISO representation of the revision date (in UTC) :>json string directory: the unique identifier that revision points to :>json string directory_url: link to :http:get:`/api/1/directory/(sha1_git)/[(path)/]` to get information about the directory associated to the revision :>json string id: the revision unique identifier :>json boolean merge: whether or not the revision corresponds to a merge commit :>json string message: the message associated to the revision :>json array parents: the parents of the revision, i.e. the previous revisions that head directly to it, each entry of that array contains an unique parent revision identifier but also a link to :http:get:`/api/1/revision/(sha1_git)/` to get more information about it :>json string type: the type of the revision -''' # noqa +""" -DOC_RETURN_REVISION_ARRAY = \ - DOC_RETURN_REVISION.replace(':>json', ':>jsonarr') +DOC_RETURN_REVISION_ARRAY = DOC_RETURN_REVISION.replace(":>json", ":>jsonarr") -def _revision_directory_by(revision, path, request_path, - limit=100, with_data=False): +def _revision_directory_by(revision, path, request_path, limit=100, with_data=False): """ Compute the revision matching criterion's directory or content data. Args: revision: dictionary of criterions representing a revision to lookup path: directory's path to lookup request_path: request path which holds the original context to limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of with_data: indicate to retrieve the content's raw data if path resolves to a content. """ + def enrich_directory_local(dir, context_url=request_path): return utils.enrich_directory(dir, context_url) rev_id, result = service.lookup_directory_through_revision( - revision, path, limit=limit, with_data=with_data) + revision, path, limit=limit, with_data=with_data + ) - content = result['content'] - if result['type'] == 'dir': # dir_entries - result['content'] = list(map(enrich_directory_local, content)) - elif result['type'] == 'file': # content - result['content'] = utils.enrich_content(content) - elif result['type'] == 'rev': # revision - result['content'] = utils.enrich_revision(content) + content = result["content"] + if result["type"] == "dir": # dir_entries + result["content"] = list(map(enrich_directory_local, content)) + elif result["type"] == "file": # content + result["content"] = utils.enrich_content(content) + elif result["type"] == "rev": # revision + result["content"] = utils.enrich_revision(content) return result -@api_route(r'/revision/(?P[0-9a-f]+)/', 'api-1-revision', - checksum_args=['sha1_git']) -@api_doc('/revision/') +@api_route( + r"/revision/(?P[0-9a-f]+)/", "api-1-revision", checksum_args=["sha1_git"] +) +@api_doc("/revision/") @format_docstring(return_revision=DOC_RETURN_REVISION) def api_revision(request, sha1_git): """ .. http:get:: /api/1/revision/(sha1_git)/ Get information about a revision in the archive. Revisions are identified by **sha1** checksums, compatible with Git commit identifiers. See :func:`swh.model.identifiers.revision_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier {common_headers} {return_revision} :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/` - """ # noqa + """ return api_lookup( - service.lookup_revision, sha1_git, - notfound_msg='Revision with sha1_git {} not found.'.format(sha1_git), + service.lookup_revision, + sha1_git, + notfound_msg="Revision with sha1_git {} not found.".format(sha1_git), enrich_fn=utils.enrich_revision, - request=request) + request=request, + ) -@api_route(r'/revision/(?P[0-9a-f]+)/raw/', - 'api-1-revision-raw-message', checksum_args=['sha1_git']) -@api_doc('/revision/raw/', tags=['hidden'], handle_response=True) +@api_route( + r"/revision/(?P[0-9a-f]+)/raw/", + "api-1-revision-raw-message", + checksum_args=["sha1_git"], +) +@api_doc("/revision/raw/", tags=["hidden"], handle_response=True) def api_revision_raw_message(request, sha1_git): """Return the raw data of the message of revision identified by sha1_git """ raw = service.lookup_revision_message(sha1_git) - response = HttpResponse(raw['message'], - content_type='application/octet-stream') - response['Content-disposition'] = \ - 'attachment;filename=rev_%s_raw' % sha1_git + response = HttpResponse(raw["message"], content_type="application/octet-stream") + response["Content-disposition"] = "attachment;filename=rev_%s_raw" % sha1_git return response -@api_route(r'/revision/(?P[0-9a-f]+)/directory/', - 'api-1-revision-directory', checksum_args=['sha1_git']) -@api_route(r'/revision/(?P[0-9a-f]+)/directory/(?P.+)/', - 'api-1-revision-directory', checksum_args=['sha1_git']) -@api_doc('/revision/directory/') +@api_route( + r"/revision/(?P[0-9a-f]+)/directory/", + "api-1-revision-directory", + checksum_args=["sha1_git"], +) +@api_route( + r"/revision/(?P[0-9a-f]+)/directory/(?P.+)/", + "api-1-revision-directory", + checksum_args=["sha1_git"], +) +@api_doc("/revision/directory/") @format_docstring() -def api_revision_directory(request, sha1_git, - dir_path=None, - with_data=False): +def api_revision_directory(request, sha1_git, dir_path=None, with_data=False): """ .. http:get:: /api/1/revision/(sha1_git)/directory/[(path)/] Get information about directory (entry) objects associated to revisions. Each revision is associated to a single "root" directory. This endpoint behaves like :http:get:`/api/1/directory/(sha1_git)/[(path)/]`, but operates on the root directory associated to a given revision. - :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier - :param string path: optional parameter to get information about the directory entry - pointed by that relative path + :param string sha1_git: hexadecimal representation of the revision + **sha1_git** identifier + :param string path: optional parameter to get information about the + directory entry pointed by that relative path {common_headers} - :>json array content: directory entries as returned by :http:get:`/api/1/directory/(sha1_git)/[(path)/]` + :>json array content: directory entries as returned by + :http:get:`/api/1/directory/(sha1_git)/[(path)/]` :>json string path: path of directory from the revision root one :>json string revision: the unique revision identifier :>json string type: the type of the directory :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`revision/f1b94134a4b879bc55c3dacdb496690c8ebdc03f/directory/` - """ # noqa - return _revision_directory_by({'sha1_git': sha1_git}, - dir_path, request.path, - with_data=with_data) - - -@api_route(r'/revision/(?P[0-9a-f]+)/log/', 'api-1-revision-log', - checksum_args=['sha1_git']) -@api_route(r'/revision/(?P[0-9a-f]+)' - r'/prev/(?P[0-9a-f]*/*)/log/', - 'api-1-revision-log', checksum_args=['sha1_git', 'prev_sha1s']) -@api_doc('/revision/log/') + """ + return _revision_directory_by( + {"sha1_git": sha1_git}, dir_path, request.path, with_data=with_data + ) + + +@api_route( + r"/revision/(?P[0-9a-f]+)/log/", + "api-1-revision-log", + checksum_args=["sha1_git"], +) +@api_route( + r"/revision/(?P[0-9a-f]+)" r"/prev/(?P[0-9a-f]*/*)/log/", + "api-1-revision-log", + checksum_args=["sha1_git", "prev_sha1s"], +) +@api_doc("/revision/log/") @format_docstring(return_revision_array=DOC_RETURN_REVISION_ARRAY) def api_revision_log(request, sha1_git, prev_sha1s=None): """ .. http:get:: /api/1/revision/(sha1_git)[/prev/(prev_sha1s)]/log/ - Get a list of all revisions heading to a given one, in other words show the commit log. + Get a list of all revisions heading to a given one, in other words show + the commit log. - :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier - :param string prev_sha1s: optional parameter representing the navigation breadcrumbs - (descendant revisions previously visited). If multiple values, use / as delimiter. - If provided, revisions information will be added at the beginning of the returned list. - :query int per_page: number of elements in the returned list, for pagination purpose + :param string sha1_git: hexadecimal representation of the revision + **sha1_git** identifier + :param string prev_sha1s: optional parameter representing the navigation + breadcrumbs (descendant revisions previously visited). If multiple values, + use / as delimiter. If provided, revisions information will be added at + the beginning of the returned list. + :query int per_page: number of elements in the returned list, for pagination + purpose {common_headers} {resheader_link} {return_revision_array} :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`revision/e1a315fa3fa734e2a6154ed7b5b9ae0eb8987aad/log/` - """ # noqa + """ result = {} - per_page = int(request.query_params.get('per_page', '10')) + per_page = int(request.query_params.get("per_page", "10")) - def lookup_revision_log_with_limit(s, limit=per_page+1): + def lookup_revision_log_with_limit(s, limit=per_page + 1): return service.lookup_revision_log(s, limit) - error_msg = 'Revision with sha1_git %s not found.' % sha1_git - rev_get = api_lookup(lookup_revision_log_with_limit, sha1_git, - notfound_msg=error_msg, - enrich_fn=utils.enrich_revision, - request=request) + error_msg = "Revision with sha1_git %s not found." % sha1_git + rev_get = api_lookup( + lookup_revision_log_with_limit, + sha1_git, + notfound_msg=error_msg, + enrich_fn=utils.enrich_revision, + request=request, + ) nb_rev = len(rev_get) - if nb_rev == per_page+1: + if nb_rev == per_page + 1: rev_backward = rev_get[:-1] - new_last_sha1 = rev_get[-1]['id'] + new_last_sha1 = rev_get[-1]["id"] query_params = {} - if request.query_params.get('per_page'): - query_params['per_page'] = per_page + if request.query_params.get("per_page"): + query_params["per_page"] = per_page - result['headers'] = { - 'link-next': reverse('api-1-revision-log', - url_args={'sha1_git': new_last_sha1}, - query_params=query_params, - request=request) + result["headers"] = { + "link-next": reverse( + "api-1-revision-log", + url_args={"sha1_git": new_last_sha1}, + query_params=query_params, + request=request, + ) } else: rev_backward = rev_get if not prev_sha1s: # no nav breadcrumbs, so we're done revisions = rev_backward else: - rev_forward_ids = prev_sha1s.split('/') + rev_forward_ids = prev_sha1s.split("/") rev_forward = api_lookup( - service.lookup_revision_multiple, rev_forward_ids, + service.lookup_revision_multiple, + rev_forward_ids, notfound_msg=error_msg, enrich_fn=utils.enrich_revision, - request=request) + request=request, + ) revisions = rev_forward + rev_backward - result.update({ - 'results': revisions - }) + result.update({"results": revisions}) return result diff --git a/swh/web/api/views/snapshot.py b/swh/web/api/views/snapshot.py index bd6ff0ad..3e2d5895 100644 --- a/swh/web/api/views/snapshot.py +++ b/swh/web/api/views/snapshot.py @@ -1,94 +1,103 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.common.utils import reverse from swh.web.config import get_config from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.utils import enrich_snapshot from swh.web.api.views.utils import api_lookup -@api_route(r'/snapshot/(?P[0-9a-f]+)/', 'api-1-snapshot', - checksum_args=['snapshot_id']) -@api_doc('/snapshot/') +@api_route( + r"/snapshot/(?P[0-9a-f]+)/", + "api-1-snapshot", + checksum_args=["snapshot_id"], +) +@api_doc("/snapshot/") @format_docstring() def api_snapshot(request, snapshot_id): """ .. http:get:: /api/1/snapshot/(snapshot_id)/ Get information about a snapshot in the archive. A snapshot is a set of named branches, which are pointers to objects at any level of the Software Heritage DAG. It represents a full picture of an origin at a given time. As well as pointing to other objects in the Software Heritage DAG, branches can also be aliases, in which case their target is the name of another branch in the same snapshot, or dangling, in which case the target is unknown. A snapshot identifier is a salted sha1. See :func:`swh.model.identifiers.snapshot_identifier` in our data model module for details about how they are computed. :param sha1 snapshot_id: a snapshot identifier :query str branches_from: optional parameter used to skip branches whose name is lesser than it before returning them :query int branches_count: optional parameter used to restrain the amount of returned branches (default to 1000) :query str target_types: optional comma separated list parameter used to filter the target types of branch to return (possible values that can be contained in that list are ``content``, ``directory``, ``revision``, ``release``, ``snapshot`` or ``alias``) {common_headers} {resheader_link} :>json object branches: object containing all branches associated to the snapshot,for each of them the associated target type and id are given but also a link to get information about that target :>json string id: the unique identifier of the snapshot :statuscode 200: no error :statuscode 400: an invalid snapshot identifier has been provided :statuscode 404: requested snapshot can not be found in the archive **Example:** .. parsed-literal:: :swh_web_api:`snapshot/6a3a2cf0b2b90ce7ae1cf0a221ed68035b686f5a/` """ - snapshot_content_max_size = get_config()['snapshot_content_max_size'] + snapshot_content_max_size = get_config()["snapshot_content_max_size"] - branches_from = request.GET.get('branches_from', '') - branches_count = int(request.GET.get('branches_count', - snapshot_content_max_size)) - target_types = request.GET.get('target_types', None) - target_types = target_types.split(',') if target_types else None + branches_from = request.GET.get("branches_from", "") + branches_count = int(request.GET.get("branches_count", snapshot_content_max_size)) + target_types = request.GET.get("target_types", None) + target_types = target_types.split(",") if target_types else None results = api_lookup( - service.lookup_snapshot, snapshot_id, branches_from, - branches_count, target_types, - notfound_msg='Snapshot with id {} not found.'.format(snapshot_id), + service.lookup_snapshot, + snapshot_id, + branches_from, + branches_count, + target_types, + notfound_msg="Snapshot with id {} not found.".format(snapshot_id), enrich_fn=enrich_snapshot, - request=request) - - response = {'results': results, 'headers': {}} - - if results['next_branch'] is not None: - response['headers']['link-next'] = reverse( - 'api-1-snapshot', - url_args={'snapshot_id': snapshot_id}, - query_params={'branches_from': results['next_branch'], - 'branches_count': branches_count, - 'target_types': target_types}, - request=request) + request=request, + ) + + response = {"results": results, "headers": {}} + + if results["next_branch"] is not None: + response["headers"]["link-next"] = reverse( + "api-1-snapshot", + url_args={"snapshot_id": snapshot_id}, + query_params={ + "branches_from": results["next_branch"], + "branches_count": branches_count, + "target_types": target_types, + }, + request=request, + ) return response diff --git a/swh/web/api/views/stat.py b/swh/web/api/views/stat.py index 12477717..b5409e9a 100644 --- a/swh/web/api/views/stat.py +++ b/swh/web/api/views/stat.py @@ -1,50 +1,50 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common import service from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route -@api_route(r'/stat/counters/', 'api-1-stat-counters') -@api_doc('/stat/counters/', noargs=True) +@api_route(r"/stat/counters/", "api-1-stat-counters") +@api_doc("/stat/counters/", noargs=True) @format_docstring() def api_stats(request): """ .. http:get:: /api/1/stat/counters/ Get statistics about the content of the archive. :>json number content: current number of content objects (aka files) in the archive :>json number directory: current number of directory objects in the archive :>json number origin: current number of software origins (an origin is a "place" where code source can be found, e.g. a git repository, a tarball, ...) in the archive :>json number origin_visit: current number of visits on software origins to fill the archive :>json number person: current number of persons (code source authors or committers) in the archive :>json number release: current number of releases objects in the archive :>json number revision: current number of revision objects (aka commits) in the archive :>json number skipped_content: current number of content objects (aka files) which where not inserted in the archive :>json number snapshot: current number of snapshot objects (aka set of named branches) in the archive {common_headers} :statuscode 200: no error **Example:** .. parsed-literal:: :swh_web_api:`stat/counters/` """ return service.stat_counters() diff --git a/swh/web/api/views/utils.py b/swh/web/api/views/utils.py index 5ade36c1..f9604967 100644 --- a/swh/web/api/views/utils.py +++ b/swh/web/api/views/utils.py @@ -1,86 +1,93 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from types import GeneratorType from typing import Callable, Any, Optional, Mapping, Dict from typing_extensions import Protocol from django.http import HttpRequest from rest_framework.response import Response from rest_framework.decorators import api_view from swh.web.common.exc import NotFoundExc from swh.web.api.apiurls import APIUrls, api_route class EnrichFunction(Protocol): - def __call__(self, input: Mapping[str, str], - request: Optional[HttpRequest]) -> Dict[str, str]: ... - - -def api_lookup(lookup_fn: Callable[..., Any], *args: Any, - notfound_msg: Optional[str] = 'Object not found', - enrich_fn: Optional[EnrichFunction] = None, - request: Optional[HttpRequest] = None): + def __call__( + self, input: Mapping[str, str], request: Optional[HttpRequest] + ) -> Dict[str, str]: + ... + + +def api_lookup( + lookup_fn: Callable[..., Any], + *args: Any, + notfound_msg: Optional[str] = "Object not found", + enrich_fn: Optional[EnrichFunction] = None, + request: Optional[HttpRequest] = None, +): r""" Capture a redundant behavior of: - looking up the backend with a criteria (be it an identifier or checksum) passed to the function lookup_fn - if nothing is found, raise an NotFoundExc exception with error message notfound_msg. - Otherwise if something is returned: - either as list, map or generator, map the enrich_fn function to it and return the resulting data structure as list. - either as dict and pass to enrich_fn and return the dict enriched. Args: - lookup_fn: function expects one criteria and optional supplementary \*args. - \*args: supplementary arguments to pass to lookup_fn. - notfound_msg: if nothing matching the criteria is found, raise NotFoundExc with this error message. - enrich_fn: Function to use to enrich the result returned by lookup_fn. Default to the identity function if not provided. - request: Input HTTP request that will be provided as parameter to enrich_fn. Raises: NotFoundExp or whatever `lookup_fn` raises. """ + + def _enrich_fn_noop(x, request): + return x + if enrich_fn is None: - enrich_fn = (lambda x, request: x) + enrich_fn = _enrich_fn_noop res = lookup_fn(*args) if res is None: raise NotFoundExc(notfound_msg) if isinstance(res, (list, GeneratorType)) or type(res) == map: return [enrich_fn(x, request=request) for x in res] return enrich_fn(res, request=request) -@api_view(['GET', 'HEAD']) +@api_view(["GET", "HEAD"]) def api_home(request): - return Response({}, template_name='api/api.html') + return Response({}, template_name="api/api.html") -APIUrls.add_url_pattern(r'^$', api_home, view_name='api-1-homepage') +APIUrls.add_url_pattern(r"^$", api_home, view_name="api-1-homepage") -@api_route(r'/', 'api-1-endpoints') +@api_route(r"/", "api-1-endpoints") def api_endpoints(request): """Display the list of opened api endpoints. """ routes = APIUrls.get_app_endpoints().copy() for route, doc in routes.items(): - doc['doc_intro'] = doc['docstring'].split('\n\n')[0] + doc["doc_intro"] = doc["docstring"].split("\n\n")[0] # Return a list of routes with consistent ordering - env = { - 'doc_routes': sorted(routes.items()) - } + env = {"doc_routes": sorted(routes.items())} return Response(env, template_name="api/endpoints.html") diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py index 7e5e544e..8701380b 100644 --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -1,240 +1,270 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpResponse from django.shortcuts import redirect from django.views.decorators.cache import never_cache from swh.model import hashutil from swh.web.common import service, query from swh.web.common.utils import reverse from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup # XXX: a bit spaghetti. Would be better with class-based views. def _dispatch_cook_progress(request, obj_type, obj_id): hex_id = hashutil.hash_to_hex(obj_id) - object_name = obj_type.split('_')[0].title() - if request.method == 'GET': + object_name = obj_type.split("_")[0].title() + if request.method == "GET": return api_lookup( - service.vault_progress, obj_type, obj_id, - notfound_msg=("{} '{}' was never requested." - .format(object_name, hex_id)), - request=request) - elif request.method == 'POST': - email = request.POST.get('email', request.GET.get('email', None)) + service.vault_progress, + obj_type, + obj_id, + notfound_msg=("{} '{}' was never requested.".format(object_name, hex_id)), + request=request, + ) + elif request.method == "POST": + email = request.POST.get("email", request.GET.get("email", None)) return api_lookup( - service.vault_cook, obj_type, obj_id, email, - notfound_msg=("{} '{}' not found." - .format(object_name, hex_id)), - request=request) - - -@api_route(r'/vault/directory/(?P[0-9a-f]+)/', - 'api-1-vault-cook-directory', methods=['GET', 'POST'], - checksum_args=['dir_id'], - throttle_scope='swh_vault_cooking') + service.vault_cook, + obj_type, + obj_id, + email, + notfound_msg=("{} '{}' not found.".format(object_name, hex_id)), + request=request, + ) + + +@api_route( + r"/vault/directory/(?P[0-9a-f]+)/", + "api-1-vault-cook-directory", + methods=["GET", "POST"], + checksum_args=["dir_id"], + throttle_scope="swh_vault_cooking", +) @never_cache -@api_doc('/vault/directory/') +@api_doc("/vault/directory/") @format_docstring() def api_vault_cook_directory(request, dir_id): """ .. http:get:: /api/1/vault/directory/(dir_id)/ .. http:post:: /api/1/vault/directory/(dir_id)/ Request the cooking of an archive for a directory or check its cooking status. That endpoint enables to create a vault cooking task for a directory through a POST request or check the status of a previously created one through a GET request. Once the cooking task has been executed, the resulting archive can be downloaded using the dedicated endpoint :http:get:`/api/1/vault/directory/(dir_id)/raw/`. Then to extract the cooked directory in the current one, use:: $ tar xvf path/to/directory.tar.gz :param string dir_id: the directory's sha1 identifier :query string email: e-mail to notify when the archive is ready {common_headers} :>json string fetch_url: the url from which to download the archive once it has been cooked (see :http:get:`/api/1/vault/directory/(dir_id)/raw/`) :>json string obj_type: the type of object to cook (directory or revision) :>json string progress_message: message describing the cooking task progress :>json number id: the cooking task id :>json string status: the cooking task status (either **new**, **pending**, **done** or **failed**) :>json string obj_id: the identifier of the object to cook :statuscode 200: no error :statuscode 400: an invalid directory identifier has been provided :statuscode 404: requested directory did not receive any cooking request yet (in case of GET) or can not be found in the archive (in case of POST) """ _, obj_id = query.parse_hash_with_algorithms_or_throws( - dir_id, ['sha1'], 'Only sha1_git is supported.') + dir_id, ["sha1"], "Only sha1_git is supported." + ) - res = _dispatch_cook_progress(request, 'directory', obj_id) - res['fetch_url'] = reverse('api-1-vault-fetch-directory', - url_args={'dir_id': dir_id}) + res = _dispatch_cook_progress(request, "directory", obj_id) + res["fetch_url"] = reverse( + "api-1-vault-fetch-directory", url_args={"dir_id": dir_id} + ) return res -@api_route(r'/vault/directory/(?P[0-9a-f]+)/raw/', - 'api-1-vault-fetch-directory', - checksum_args=['dir_id']) -@api_doc('/vault/directory/raw/', handle_response=True) +@api_route( + r"/vault/directory/(?P[0-9a-f]+)/raw/", + "api-1-vault-fetch-directory", + checksum_args=["dir_id"], +) +@api_doc("/vault/directory/raw/", handle_response=True) def api_vault_fetch_directory(request, dir_id): """ .. http:get:: /api/1/vault/directory/(dir_id)/raw/ Fetch the cooked archive for a directory. See :http:get:`/api/1/vault/directory/(dir_id)/` to get more details on directory cooking. :param string dir_id: the directory's sha1 identifier :resheader Content-Type: application/octet-stream :statuscode 200: no error :statuscode 400: an invalid directory identifier has been provided :statuscode 404: requested directory did not receive any cooking request yet (in case of GET) or can not be found in the archive (in case of POST) """ _, obj_id = query.parse_hash_with_algorithms_or_throws( - dir_id, ['sha1'], 'Only sha1_git is supported.') + dir_id, ["sha1"], "Only sha1_git is supported." + ) res = api_lookup( - service.vault_fetch, 'directory', obj_id, + service.vault_fetch, + "directory", + obj_id, notfound_msg="Directory with ID '{}' not found.".format(dir_id), - request=request) - fname = '{}.tar.gz'.format(dir_id) - response = HttpResponse(res, content_type='application/gzip') - response['Content-disposition'] = 'attachment; filename={}'.format(fname) + request=request, + ) + fname = "{}.tar.gz".format(dir_id) + response = HttpResponse(res, content_type="application/gzip") + response["Content-disposition"] = "attachment; filename={}".format(fname) return response -@api_route(r'/vault/revision/(?P[0-9a-f]+)/gitfast/', - 'api-1-vault-cook-revision_gitfast', methods=['GET', 'POST'], - checksum_args=['rev_id'], - throttle_scope='swh_vault_cooking') +@api_route( + r"/vault/revision/(?P[0-9a-f]+)/gitfast/", + "api-1-vault-cook-revision_gitfast", + methods=["GET", "POST"], + checksum_args=["rev_id"], + throttle_scope="swh_vault_cooking", +) @never_cache -@api_doc('/vault/revision/gitfast/') +@api_doc("/vault/revision/gitfast/") @format_docstring() def api_vault_cook_revision_gitfast(request, rev_id): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/ .. http:post:: /api/1/vault/revision/(rev_id)/gitfast/ Request the cooking of a gitfast archive for a revision or check its cooking status. That endpoint enables to create a vault cooking task for a revision through a POST request or check the status of a previously created one through a GET request. Once the cooking task has been executed, the resulting gitfast archive can be downloaded using the dedicated endpoint :http:get:`/api/1/vault/revision/(rev_id)/gitfast/raw/`. Then to import the revision in the current directory, use:: $ git init $ zcat path/to/revision.gitfast.gz | git fast-import $ git checkout HEAD :param string rev_id: the revision's sha1 identifier :query string email: e-mail to notify when the gitfast archive is ready {common_headers} :>json string fetch_url: the url from which to download the archive once it has been cooked (see :http:get:`/api/1/vault/revision/(rev_id)/gitfast/raw/`) :>json string obj_type: the type of object to cook (directory or revision) :>json string progress_message: message describing the cooking task progress :>json number id: the cooking task id :>json string status: the cooking task status (new/pending/done/failed) :>json string obj_id: the identifier of the object to cook :statuscode 200: no error :statuscode 400: an invalid revision identifier has been provided :statuscode 404: requested directory did not receive any cooking request yet (in case of GET) or can not be found in the archive (in case of POST) """ _, obj_id = query.parse_hash_with_algorithms_or_throws( - rev_id, ['sha1'], 'Only sha1_git is supported.') + rev_id, ["sha1"], "Only sha1_git is supported." + ) - res = _dispatch_cook_progress(request, 'revision_gitfast', obj_id) - res['fetch_url'] = reverse('api-1-vault-fetch-revision_gitfast', - url_args={'rev_id': rev_id}) + res = _dispatch_cook_progress(request, "revision_gitfast", obj_id) + res["fetch_url"] = reverse( + "api-1-vault-fetch-revision_gitfast", url_args={"rev_id": rev_id} + ) return res -@api_route(r'/vault/revision/(?P[0-9a-f]+)/gitfast/raw/', - 'api-1-vault-fetch-revision_gitfast', - checksum_args=['rev_id']) -@api_doc('/vault/revision/gitfast/raw/', handle_response=True) +@api_route( + r"/vault/revision/(?P[0-9a-f]+)/gitfast/raw/", + "api-1-vault-fetch-revision_gitfast", + checksum_args=["rev_id"], +) +@api_doc("/vault/revision/gitfast/raw/", handle_response=True) def api_vault_fetch_revision_gitfast(request, rev_id): """ .. http:get:: /api/1/vault/revision/(rev_id)/gitfast/raw/ Fetch the cooked gitfast archive for a revision. See :http:get:`/api/1/vault/revision/(rev_id)/gitfast/` to get more details on directory cooking. :param string rev_id: the revision's sha1 identifier :resheader Content-Type: application/octet-stream :statuscode 200: no error :statuscode 400: an invalid revision identifier has been provided :statuscode 404: requested directory did not receive any cooking request yet (in case of GET) or can not be found in the archive (in case of POST) """ _, obj_id = query.parse_hash_with_algorithms_or_throws( - rev_id, ['sha1'], 'Only sha1_git is supported.') + rev_id, ["sha1"], "Only sha1_git is supported." + ) res = api_lookup( - service.vault_fetch, 'revision_gitfast', obj_id, + service.vault_fetch, + "revision_gitfast", + obj_id, notfound_msg="Revision with ID '{}' not found.".format(rev_id), - request=request) - fname = '{}.gitfast.gz'.format(rev_id) - response = HttpResponse(res, content_type='application/gzip') - response['Content-disposition'] = 'attachment; filename={}'.format(fname) + request=request, + ) + fname = "{}.gitfast.gz".format(rev_id) + response = HttpResponse(res, content_type="application/gzip") + response["Content-disposition"] = "attachment; filename={}".format(fname) return response -@api_route(r'/vault/revision_gitfast/(?P[0-9a-f]+)/raw/', - 'api-1-vault-revision_gitfast-raw', - checksum_args=['rev_id']) -@api_doc('/vault/revision_gitfast/raw/', tags=['hidden'], handle_response=True) +@api_route( + r"/vault/revision_gitfast/(?P[0-9a-f]+)/raw/", + "api-1-vault-revision_gitfast-raw", + checksum_args=["rev_id"], +) +@api_doc("/vault/revision_gitfast/raw/", tags=["hidden"], handle_response=True) def _api_vault_revision_gitfast_raw(request, rev_id): """ The vault backend sends an email containing an invalid url to fetch a gitfast archive. So setup a redirection to the correct one as a temporary workaround. """ - rev_gitfast_raw_url = reverse('api-1-vault-fetch-revision_gitfast', - url_args={'rev_id': rev_id}) + rev_gitfast_raw_url = reverse( + "api-1-vault-fetch-revision_gitfast", url_args={"rev_id": rev_id} + ) return redirect(rev_gitfast_raw_url) diff --git a/swh/web/auth/backends.py b/swh/web/auth/backends.py index 004bf84c..9c9cfb03 100644 --- a/swh/web/auth/backends.py +++ b/swh/web/auth/backends.py @@ -1,145 +1,145 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timedelta from typing import Any, Dict, Optional from django.core.cache import cache from django.http import HttpRequest from django.utils import timezone from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed import sentry_sdk from swh.web.auth.keycloak import KeycloakOpenIDConnect from swh.web.auth.utils import get_oidc_client from swh.web.auth.models import OIDCUser # OpenID Connect client to communicate with Keycloak server _oidc_client: KeycloakOpenIDConnect = get_oidc_client() def _oidc_user_from_decoded_token(decoded_token: Dict[str, Any]) -> OIDCUser: # compute an integer user identifier for Django User model # by concatenating all groups of the UUID4 user identifier # generated by Keycloak and converting it from hex to decimal - user_id = int(''.join(decoded_token['sub'].split('-')), 16) + user_id = int("".join(decoded_token["sub"].split("-")), 16) # create a Django user that will not be saved to database - user = OIDCUser(id=user_id, - username=decoded_token['preferred_username'], - password='', - first_name=decoded_token['given_name'], - last_name=decoded_token['family_name'], - email=decoded_token['email']) + user = OIDCUser( + id=user_id, + username=decoded_token["preferred_username"], + password="", + first_name=decoded_token["given_name"], + last_name=decoded_token["family_name"], + email=decoded_token["email"], + ) # set is_staff user property based on groups - if 'groups' in decoded_token: - user.is_staff = '/staff' in decoded_token['groups'] + if "groups" in decoded_token: + user.is_staff = "/staff" in decoded_token["groups"] # add user sub to custom User proxy model - user.sub = decoded_token['sub'] + user.sub = decoded_token["sub"] return user def _oidc_user_from_profile(oidc_profile: Dict[str, Any]) -> OIDCUser: # decode JWT token - decoded_token = _oidc_client.decode_token(oidc_profile['access_token']) + decoded_token = _oidc_client.decode_token(oidc_profile["access_token"]) # create OIDCUser from decoded token user = _oidc_user_from_decoded_token(decoded_token) # get authentication init datetime - auth_datetime = datetime.fromtimestamp(decoded_token['auth_time']) - exp_datetime = datetime.fromtimestamp(decoded_token['exp']) + auth_datetime = datetime.fromtimestamp(decoded_token["auth_time"]) + exp_datetime = datetime.fromtimestamp(decoded_token["exp"]) # compute OIDC tokens expiration date - oidc_profile['expires_at'] = exp_datetime - oidc_profile['refresh_expires_at'] = ( - auth_datetime + - timedelta(seconds=oidc_profile['refresh_expires_in'])) + oidc_profile["expires_at"] = exp_datetime + oidc_profile["refresh_expires_at"] = auth_datetime + timedelta( + seconds=oidc_profile["refresh_expires_in"] + ) # add OIDC profile data to custom User proxy model for key, val in oidc_profile.items(): if hasattr(user, key): setattr(user, key, val) return user class OIDCAuthorizationCodePKCEBackend: - - def authenticate(self, request: HttpRequest, code: str, code_verifier: str, - redirect_uri: str) -> Optional[OIDCUser]: + def authenticate( + self, request: HttpRequest, code: str, code_verifier: str, redirect_uri: str + ) -> Optional[OIDCUser]: user = None try: # try to authenticate user with OIDC PKCE authorization code flow oidc_profile = _oidc_client.authorization_code( - code, redirect_uri, code_verifier=code_verifier) + code, redirect_uri, code_verifier=code_verifier + ) # create Django user user = _oidc_user_from_profile(oidc_profile) # set cache key TTL as access token expiration time assert user.expires_at ttl = int(user.expires_at.timestamp() - timezone.now().timestamp()) # save oidc_profile in cache - cache.set(f'oidc_user_{user.id}', oidc_profile, - timeout=max(0, ttl)) + cache.set(f"oidc_user_{user.id}", oidc_profile, timeout=max(0, ttl)) except Exception as e: sentry_sdk.capture_exception(e) return user def get_user(self, user_id: int) -> Optional[OIDCUser]: # get oidc profile from cache - oidc_profile = cache.get(f'oidc_user_{user_id}') + oidc_profile = cache.get(f"oidc_user_{user_id}") if oidc_profile: try: user = _oidc_user_from_profile(oidc_profile) # restore auth backend - setattr(user, 'backend', - f'{__name__}.{self.__class__.__name__}') + setattr(user, "backend", f"{__name__}.{self.__class__.__name__}") return user except Exception as e: sentry_sdk.capture_exception(e) return None else: return None class OIDCBearerTokenAuthentication(BaseAuthentication): def authenticate(self, request): - auth_header = request.META.get('HTTP_AUTHORIZATION') + auth_header = request.META.get("HTTP_AUTHORIZATION") if auth_header is None: return None try: - auth_type, token = auth_header.split(' ', 1) + auth_type, token = auth_header.split(" ", 1) except ValueError: - raise AuthenticationFailed( - 'Invalid HTTP authorization header format') + raise AuthenticationFailed("Invalid HTTP authorization header format") - if auth_type != 'Bearer': + if auth_type != "Bearer": raise AuthenticationFailed( - (f'Invalid or unsupported HTTP authorization' - f' type ({auth_type}).')) + (f"Invalid or unsupported HTTP authorization" f" type ({auth_type}).") + ) try: # attempt to decode token decoded_token = _oidc_client.decode_token(token) # create Django user user = _oidc_user_from_decoded_token(decoded_token) except Exception as e: sentry_sdk.capture_exception(e) raise AuthenticationFailed(str(e)) return user, None diff --git a/swh/web/auth/keycloak.py b/swh/web/auth/keycloak.py index bf4c13fc..4e5fa2af 100644 --- a/swh/web/auth/keycloak.py +++ b/swh/web/auth/keycloak.py @@ -1,162 +1,168 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Dict, Optional, Tuple from urllib.parse import urlencode from keycloak import KeycloakOpenID class KeycloakOpenIDConnect: """ Wrapper class around python-keycloak to ease the interaction with Keycloak for managing authentication and user permissions with OpenID Connect. """ - def __init__(self, server_url: str, realm_name: str, client_id: str, - realm_public_key: str = ''): + def __init__( + self, + server_url: str, + realm_name: str, + client_id: str, + realm_public_key: str = "", + ): """ Args: server_url: URL of the Keycloak server realm_name: The realm name client_id: The OpenID Connect client identifier realm_public_key: The realm public key (will be dynamically retrieved if not provided) """ self._keycloak = KeycloakOpenID( - server_url=server_url, - client_id=client_id, - realm_name=realm_name, + server_url=server_url, client_id=client_id, realm_name=realm_name, ) self.server_url = server_url self.realm_name = realm_name self.client_id = client_id self.realm_public_key = realm_public_key def well_known(self) -> Dict[str, Any]: """ Retrieve the OpenID Connect Well-Known URI registry from Keycloak. Returns: A dictionary filled with OpenID Connect URIS. """ return self._keycloak.well_know() - def authorization_url(self, redirect_uri: str, - **extra_params: str) -> str: + def authorization_url(self, redirect_uri: str, **extra_params: str) -> str: """ Get OpenID Connect authorization URL to authenticate users. Args: redirect_uri: URI to redirect to once a user is authenticated extra_params: Extra query parameters to add to the authorization URL """ auth_url = self._keycloak.auth_url(redirect_uri) if extra_params: - auth_url += '&%s' % urlencode(extra_params) + auth_url += "&%s" % urlencode(extra_params) return auth_url - def authorization_code(self, code: str, redirect_uri: str, - **extra_params: str) -> Dict[str, Any]: + def authorization_code( + self, code: str, redirect_uri: str, **extra_params: str + ) -> Dict[str, Any]: """ Get OpenID Connect authentication tokens using Authorization Code flow. Args: code: Authorization code provided by Keycloak redirect_uri: URI to redirect to once a user is authenticated (must be the same as the one provided to authorization_url) extra_params: Extra parameters to add in the authorization request payload. """ return self._keycloak.token( - grant_type='authorization_code', + grant_type="authorization_code", code=code, redirect_uri=redirect_uri, - **extra_params) + **extra_params, + ) def refresh_token(self, refresh_token: str) -> Dict[str, Any]: """ Request a new access token from Keycloak using a refresh token. Args: refresh_token: A refresh token provided by Keycloak Returns: A dictionary filled with tokens info """ return self._keycloak.refresh_token(refresh_token) - def decode_token(self, token: str, - options: Optional[Dict[str, Any]] = None - ) -> Dict[str, Any]: + def decode_token( + self, token: str, options: Optional[Dict[str, Any]] = None + ) -> Dict[str, Any]: """ Try to decode a JWT token. Args: token: A JWT token to decode options: Options for jose.jwt.decode Returns: A dictionary filled with decoded token content """ if not self.realm_public_key: realm_public_key = self._keycloak.public_key() - self.realm_public_key = '-----BEGIN PUBLIC KEY-----\n' + self.realm_public_key = "-----BEGIN PUBLIC KEY-----\n" self.realm_public_key += realm_public_key - self.realm_public_key += '\n-----END PUBLIC KEY-----' + self.realm_public_key += "\n-----END PUBLIC KEY-----" - return self._keycloak.decode_token(token, key=self.realm_public_key, - options=options) + return self._keycloak.decode_token( + token, key=self.realm_public_key, options=options + ) def logout(self, refresh_token: str) -> None: """ Logout a user by closing its authenticated session. Args: refresh_token: A refresh token provided by Keycloak """ self._keycloak.logout(refresh_token) def userinfo(self, access_token: str) -> Dict[str, Any]: """ Return user information from its access token. Args: access_token: An access token provided by Keycloak Returns: A dictionary fillled with user information """ return self._keycloak.userinfo(access_token) # stores instances of KeycloakOpenIDConnect class # dict keys are (realm_name, client_id) tuples _keycloak_oidc: Dict[Tuple[str, str], KeycloakOpenIDConnect] = {} -def get_keycloak_oidc_client(server_url: str, realm_name: str, - client_id: str) -> KeycloakOpenIDConnect: +def get_keycloak_oidc_client( + server_url: str, realm_name: str, client_id: str +) -> KeycloakOpenIDConnect: """ Instantiate a KeycloakOpenIDConnect class for a given client in a given realm. Args: server_url: Base URL of a Keycloak server realm_name: Name of the realm in Keycloak client_id: Client identifier in the realm Returns: An object to ease the interaction with the Keycloak server """ realm_client_key = (realm_name, client_id) if realm_client_key not in _keycloak_oidc: - _keycloak_oidc[realm_client_key] = KeycloakOpenIDConnect(server_url, - realm_name, - client_id) + _keycloak_oidc[realm_client_key] = KeycloakOpenIDConnect( + server_url, realm_name, client_id + ) return _keycloak_oidc[realm_client_key] diff --git a/swh/web/auth/middlewares.py b/swh/web/auth/middlewares.py index 0664c368..f750138e 100644 --- a/swh/web/auth/middlewares.py +++ b/swh/web/auth/middlewares.py @@ -1,43 +1,44 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.contrib.auth import BACKEND_SESSION_KEY from django.http.response import HttpResponseRedirect from swh.web.common.utils import reverse class OIDCSessionRefreshMiddleware: """ Middleware for silently refreshing on OpenID Connect session from the browser and get new access token. """ + def __init__(self, get_response=None): self.get_response = get_response self.exempted_urls = [ - reverse(v) for v in ('logout', - 'oidc-login', - 'oidc-login-complete', - 'oidc-logout') + reverse(v) + for v in ("logout", "oidc-login", "oidc-login-complete", "oidc-logout") ] def __call__(self, request): - if (request.method != 'GET' - or request.user.is_authenticated - or BACKEND_SESSION_KEY not in request.session - or 'OIDC' not in request.session[BACKEND_SESSION_KEY] - or request.path in self.exempted_urls): + if ( + request.method != "GET" + or request.user.is_authenticated + or BACKEND_SESSION_KEY not in request.session + or "OIDC" not in request.session[BACKEND_SESSION_KEY] + or request.path in self.exempted_urls + ): return self.get_response(request) # At that point, we know that a OIDC user was previously logged in. # Access token has expired so we attempt a silent OIDC session refresh. # If the latter failed because the session expired, user will be # redirected to logout page and a link will be offered to login again. # See implementation of "oidc-login-complete" view for more details. next_path = request.get_full_path() - redirect_url = reverse('oidc-login', - query_params={'next_path': next_path, - 'prompt': 'none'}) + redirect_url = reverse( + "oidc-login", query_params={"next_path": next_path, "prompt": "none"} + ) return HttpResponseRedirect(redirect_url) diff --git a/swh/web/auth/models.py b/swh/web/auth/models.py index 91d7c2ee..01783857 100644 --- a/swh/web/auth/models.py +++ b/swh/web/auth/models.py @@ -1,43 +1,43 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from typing import Optional from django.contrib.auth.models import User class OIDCUser(User): """ Custom User proxy model for remote users storing OpenID Connect related data: profile containing authentication tokens. The model is also not saved to database as all users are already stored in the Keycloak one. """ # OIDC subject identifier - sub: str = '' + sub: str = "" # OIDC tokens and session related data, only relevant when a user # authenticates from a web browser access_token: Optional[str] = None expires_at: Optional[datetime] = None id_token: Optional[str] = None refresh_token: Optional[str] = None refresh_expires_at: Optional[datetime] = None scope: Optional[str] = None session_state: Optional[str] = None class Meta: - app_label = 'swh.web.auth' + app_label = "swh.web.auth" proxy = True def save(self, **kwargs): """ Override django.db.models.Model.save to avoid saving the remote users to web application database. """ pass diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py index bda492d3..829b2450 100644 --- a/swh/web/auth/utils.py +++ b/swh/web/auth/utils.py @@ -1,62 +1,61 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import secrets from base64 import urlsafe_b64encode from typing import Tuple -from swh.web.auth.keycloak import ( - KeycloakOpenIDConnect, get_keycloak_oidc_client -) +from swh.web.auth.keycloak import KeycloakOpenIDConnect, get_keycloak_oidc_client from swh.web.config import get_config def gen_oidc_pkce_codes() -> Tuple[str, str]: """ Generates a code verifier and a code challenge to be used with the OpenID Connect authorization code flow with PKCE ("Proof Key for Code Exchange", see https://tools.ietf.org/html/rfc7636). PKCE replaces the static secret used in the standard authorization code flow with a temporary one-time challenge, making it feasible to use in public clients. The implementation is inspired from that blog post: https://www.stefaanlippens.net/oauth-code-flow-pkce.html """ # generate a code verifier which is a long enough random alphanumeric # string, only to be used "client side" code_verifier_str = secrets.token_urlsafe(60) # create the PKCE code challenge by hashing the code verifier with SHA256 # and encoding the result in URL-safe base64 (without padding) - code_challenge = hashlib.sha256(code_verifier_str.encode('ascii')).digest() - code_challenge_str = urlsafe_b64encode(code_challenge).decode('ascii') - code_challenge_str = code_challenge_str.replace('=', '') + code_challenge = hashlib.sha256(code_verifier_str.encode("ascii")).digest() + code_challenge_str = urlsafe_b64encode(code_challenge).decode("ascii") + code_challenge_str = code_challenge_str.replace("=", "") return code_verifier_str, code_challenge_str -OIDC_SWH_WEB_CLIENT_ID = 'swh-web' +OIDC_SWH_WEB_CLIENT_ID = "swh-web" -def get_oidc_client(client_id: str = OIDC_SWH_WEB_CLIENT_ID - ) -> KeycloakOpenIDConnect: +def get_oidc_client(client_id: str = OIDC_SWH_WEB_CLIENT_ID) -> KeycloakOpenIDConnect: """ Instantiate a KeycloakOpenIDConnect class for a given client in the SoftwareHeritage realm. Args: client_id: client identifier in the SoftwareHeritage realm Returns: An object to ease the interaction with the Keycloak server """ swhweb_config = get_config() - return get_keycloak_oidc_client(swhweb_config['keycloak']['server_url'], - swhweb_config['keycloak']['realm_name'], - client_id) + return get_keycloak_oidc_client( + swhweb_config["keycloak"]["server_url"], + swhweb_config["keycloak"]["realm_name"], + client_id, + ) diff --git a/swh/web/auth/views.py b/swh/web/auth/views.py index 85c211d3..0690dfda 100644 --- a/swh/web/auth/views.py +++ b/swh/web/auth/views.py @@ -1,133 +1,137 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid from typing import cast from django.conf.urls import url from django.core.cache import cache from django.contrib.auth import authenticate, login, logout from django.http import HttpRequest from django.http.response import ( - HttpResponse, HttpResponseRedirect, HttpResponseServerError + HttpResponse, + HttpResponseRedirect, + HttpResponseServerError, ) from swh.web.auth.models import OIDCUser from swh.web.auth.utils import gen_oidc_pkce_codes, get_oidc_client from swh.web.common.exc import handle_view_exception, BadInputExc from swh.web.common.utils import reverse def oidc_login(request: HttpRequest) -> HttpResponse: """ Django view to initiate login process using OpenID Connect. """ # generate a CSRF token state = str(uuid.uuid4()) - redirect_uri = reverse('oidc-login-complete', request=request) + redirect_uri = reverse("oidc-login-complete", request=request) code_verifier, code_challenge = gen_oidc_pkce_codes() - request.session['login_data'] = { - 'code_verifier': code_verifier, - 'state': state, - 'redirect_uri': redirect_uri, - 'next_path': request.GET.get('next_path', ''), - 'prompt': request.GET.get('prompt', ''), + request.session["login_data"] = { + "code_verifier": code_verifier, + "state": state, + "redirect_uri": redirect_uri, + "next_path": request.GET.get("next_path", ""), + "prompt": request.GET.get("prompt", ""), } authorization_url_params = { - 'state': state, - 'code_challenge': code_challenge, - 'code_challenge_method': 'S256', - 'scope': 'openid', - 'prompt': request.GET.get('prompt', ''), + "state": state, + "code_challenge": code_challenge, + "code_challenge_method": "S256", + "scope": "openid", + "prompt": request.GET.get("prompt", ""), } try: oidc_client = get_oidc_client() authorization_url = oidc_client.authorization_url( - redirect_uri, **authorization_url_params) + redirect_uri, **authorization_url_params + ) return HttpResponseRedirect(authorization_url) except Exception as e: return handle_view_exception(request, e) def oidc_login_complete(request: HttpRequest) -> HttpResponse: """ Django view to finalize login process using OpenID Connect. """ try: - if 'login_data' not in request.session: - raise Exception('Login process has not been initialized.') + if "login_data" not in request.session: + raise Exception("Login process has not been initialized.") - login_data = request.session['login_data'] - next_path = login_data['next_path'] or request.build_absolute_uri('/') + login_data = request.session["login_data"] + next_path = login_data["next_path"] or request.build_absolute_uri("/") - if 'error' in request.GET: - if login_data['prompt'] == 'none': + if "error" in request.GET: + if login_data["prompt"] == "none": # Silent login failed because OIDC session expired. # Redirect to logout page and inform user. logout(request) - logout_url = reverse('logout', - query_params={'next_path': next_path, - 'remote_user': 1}) + logout_url = reverse( + "logout", query_params={"next_path": next_path, "remote_user": 1} + ) return HttpResponseRedirect(logout_url) - return HttpResponseServerError(request.GET['error']) + return HttpResponseServerError(request.GET["error"]) - if 'code' not in request.GET or 'state' not in request.GET: - raise BadInputExc('Missing query parameters for authentication.') + if "code" not in request.GET or "state" not in request.GET: + raise BadInputExc("Missing query parameters for authentication.") # get CSRF token returned by OIDC server - state = request.GET['state'] + state = request.GET["state"] - if state != login_data['state']: - raise BadInputExc('Wrong CSRF token, aborting login process.') + if state != login_data["state"]: + raise BadInputExc("Wrong CSRF token, aborting login process.") - user = authenticate(request=request, - code=request.GET['code'], - code_verifier=login_data['code_verifier'], - redirect_uri=login_data['redirect_uri']) + user = authenticate( + request=request, + code=request.GET["code"], + code_verifier=login_data["code_verifier"], + redirect_uri=login_data["redirect_uri"], + ) if user is None: - raise Exception('User authentication failed.') + raise Exception("User authentication failed.") login(request, user) return HttpResponseRedirect(next_path) except Exception as e: return handle_view_exception(request, e) def oidc_logout(request: HttpRequest) -> HttpResponse: """ Django view to logout using OpenID Connect. """ try: user = request.user logout(request) - if hasattr(user, 'refresh_token'): + if hasattr(user, "refresh_token"): oidc_client = get_oidc_client() user = cast(OIDCUser, user) refresh_token = cast(str, user.refresh_token) # end OpenID Connect session oidc_client.logout(refresh_token) # remove user data from cache - cache.delete(f'oidc_user_{user.id}') + cache.delete(f"oidc_user_{user.id}") - logout_url = reverse('logout', query_params={'remote_user': 1}) + logout_url = reverse("logout", query_params={"remote_user": 1}) return HttpResponseRedirect(request.build_absolute_uri(logout_url)) except Exception as e: return handle_view_exception(request, e) urlpatterns = [ - url(r'^oidc/login/$', oidc_login, name='oidc-login'), - url(r'^oidc/login-complete/$', oidc_login_complete, - name='oidc-login-complete'), - url(r'^oidc/logout/$', oidc_logout, name='oidc-logout'), + url(r"^oidc/login/$", oidc_login, name="oidc-login"), + url(r"^oidc/login-complete/$", oidc_login_complete, name="oidc-login-complete"), + url(r"^oidc/logout/$", oidc_logout, name="oidc-logout"), ] diff --git a/swh/web/browse/browseurls.py b/swh/web/browse/browseurls.py index 7330ce4c..08afd147 100644 --- a/swh/web/browse/browseurls.py +++ b/swh/web/browse/browseurls.py @@ -1,42 +1,42 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.urlsindex import UrlsIndex class BrowseUrls(UrlsIndex): """ Class to manage swh-web browse application urls. """ - scope = 'browse' + scope = "browse" def browse_route(*url_patterns, view_name=None, checksum_args=None): """ Decorator to ease the registration of a swh-web browse endpoint Args: url_patterns: list of url patterns used by Django to identify the browse routes view_name: the name of the Django view associated to the routes used to reverse the url """ - url_patterns = ['^' + url_pattern + '$' for url_pattern in url_patterns] + url_patterns = ["^" + url_pattern + "$" for url_pattern in url_patterns] view_name = view_name def decorator(f): # register the route and its view in the browse endpoints index for url_pattern in url_patterns: BrowseUrls.add_url_pattern(url_pattern, f, view_name) if checksum_args: - BrowseUrls.add_redirect_for_checksum_args(view_name, - url_patterns, - checksum_args) + BrowseUrls.add_redirect_for_checksum_args( + view_name, url_patterns, checksum_args + ) return f return decorator diff --git a/swh/web/browse/identifiers.py b/swh/web/browse/identifiers.py index fc81257b..be9a0070 100644 --- a/swh/web/browse/identifiers.py +++ b/swh/web/browse/identifiers.py @@ -1,25 +1,24 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import redirect from swh.web.common.identifiers import resolve_swh_persistent_id from swh.web.common.exc import handle_view_exception def swh_id_browse(request, swh_id): """ Django view enabling to browse the archive using :ref:`persistent-identifiers`. The url that points to it is :http:get:`/(swh_id)/`. """ try: - swh_id_resolved = resolve_swh_persistent_id( - swh_id, query_params=request.GET) + swh_id_resolved = resolve_swh_persistent_id(swh_id, query_params=request.GET) except Exception as exc: return handle_view_exception(request, exc) - return redirect(swh_id_resolved['browse_url']) + return redirect(swh_id_resolved["browse_url"]) diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py index 2fe4a0c6..f802ef5d 100644 --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -1,51 +1,55 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url from django.shortcuts import render, redirect -import swh.web.browse.views.directory # noqa -import swh.web.browse.views.content # noqa -import swh.web.browse.views.origin # noqa -import swh.web.browse.views.release # noqa -import swh.web.browse.views.revision # noqa -import swh.web.browse.views.snapshot # noqa +import swh.web.browse.views.directory # noqa +import swh.web.browse.views.content # noqa +import swh.web.browse.views.origin # noqa +import swh.web.browse.views.release # noqa +import swh.web.browse.views.revision # noqa +import swh.web.browse.views.snapshot # noqa from swh.web.browse.browseurls import BrowseUrls from swh.web.browse.identifiers import swh_id_browse from swh.web.common.utils import reverse def _browse_help_view(request): - return render(request, 'browse/help.html', - {'heading': 'How to browse the archive ?'}) + return render( + request, "browse/help.html", {"heading": "How to browse the archive ?"} + ) def _browse_search_view(request): - return render(request, 'browse/search.html', - {'heading': 'Search software origins to browse'}) + return render( + request, "browse/search.html", {"heading": "Search software origins to browse"} + ) def _browse_vault_view(request): - return render(request, 'browse/vault-ui.html', - {'heading': 'Download archive content from the Vault'}) + return render( + request, + "browse/vault-ui.html", + {"heading": "Download archive content from the Vault"}, + ) def _browse_origin_save_view(request): - return redirect(reverse('origin-save')) + return redirect(reverse("origin-save")) urlpatterns = [ - url(r'^$', _browse_search_view), - url(r'^help/$', _browse_help_view, name='browse-help'), - url(r'^search/$', _browse_search_view, name='browse-search'), - url(r'^vault/$', _browse_vault_view, name='browse-vault'), + url(r"^$", _browse_search_view), + url(r"^help/$", _browse_help_view, name="browse-help"), + url(r"^search/$", _browse_search_view, name="browse-search"), + url(r"^vault/$", _browse_vault_view, name="browse-vault"), # for backward compatibility - url(r'^origin/save/$', _browse_origin_save_view, - name='browse-origin-save'), - url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse), + url(r"^origin/save/$", _browse_origin_save_view, name="browse-origin-save"), + url(r"^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$", swh_id_browse), ] urlpatterns += BrowseUrls.get_url_patterns() diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py index bbf6036f..052ddca9 100644 --- a/swh/web/browse/utils.py +++ b/swh/web/browse/utils.py @@ -1,1105 +1,1137 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import base64 import magic import stat import textwrap from collections import defaultdict from threading import Lock from django.core.cache import cache from django.utils.safestring import mark_safe from django.utils.html import escape import sentry_sdk from swh.model.identifiers import persistent_identifier from swh.web.common import highlightjs, service from swh.web.common.exc import NotFoundExc, http_status_code_message from swh.web.common.identifiers import get_swh_persistent_id from swh.web.common.origin_visits import get_origin_visit from swh.web.common.utils import ( - reverse, format_utc_iso_date, swh_object_icons, rst_to_html + reverse, + format_utc_iso_date, + swh_object_icons, + rst_to_html, ) from swh.web.config import get_config def get_directory_entries(sha1_git): """Function that retrieves the content of a directory from the archive. The directories entries are first sorted in lexicographical order. Sub-directories and regular files are then extracted. Args: sha1_git: sha1_git identifier of the directory Returns: A tuple whose first member corresponds to the sub-directories list and second member the regular files list Raises: NotFoundExc if the directory is not found """ - cache_entry_id = 'directory_entries_%s' % sha1_git + cache_entry_id = "directory_entries_%s" % sha1_git cache_entry = cache.get(cache_entry_id) if cache_entry: return cache_entry entries = list(service.lookup_directory(sha1_git)) for e in entries: - e['perms'] = stat.filemode(e['perms']) - if e['type'] == 'rev': + e["perms"] = stat.filemode(e["perms"]) + if e["type"] == "rev": # modify dir entry name to explicitly show it points # to a revision - e['name'] = '%s @ %s' % (e['name'], e['target'][:7]) + e["name"] = "%s @ %s" % (e["name"], e["target"][:7]) - dirs = [e for e in entries if e['type'] in ('dir', 'rev')] - files = [e for e in entries if e['type'] == 'file'] + dirs = [e for e in entries if e["type"] in ("dir", "rev")] + files = [e for e in entries if e["type"] == "file"] - dirs = sorted(dirs, key=lambda d: d['name']) - files = sorted(files, key=lambda f: f['name']) + dirs = sorted(dirs, key=lambda d: d["name"]) + files = sorted(files, key=lambda f: f["name"]) cache.set(cache_entry_id, (dirs, files)) return dirs, files _lock = Lock() def get_mimetype_and_encoding_for_content(content): """Function that returns the mime type and the encoding associated to a content buffer using the magic module under the hood. Args: content (bytes): a content buffer Returns: A tuple (mimetype, encoding), for instance ('text/plain', 'us-ascii'), associated to the provided content. """ # https://pypi.org/project/python-magic/ # packaged as python3-magic in debian buster - if hasattr(magic, 'from_buffer'): + if hasattr(magic, "from_buffer"): m = magic.Magic(mime=True, mime_encoding=True) mime_encoding = m.from_buffer(content) - mime_type, encoding = mime_encoding.split(';') - encoding = encoding.replace(' charset=', '') + mime_type, encoding = mime_encoding.split(";") + encoding = encoding.replace(" charset=", "") # https://pypi.org/project/file-magic/ # packaged as python3-magic in debian stretch else: # TODO: Remove that code when production environment is upgraded # to debian buster # calls to the file-magic API are not thread-safe so they must # be protected with a Lock to guarantee they will succeed _lock.acquire() magic_result = magic.detect_from_content(content) _lock.release() mime_type = magic_result.mime_type encoding = magic_result.encoding return mime_type, encoding # maximum authorized content size in bytes for HTML display # with code highlighting -content_display_max_size = get_config()['content_display_max_size'] +content_display_max_size = get_config()["content_display_max_size"] -snapshot_content_max_size = get_config()['snapshot_content_max_size'] +snapshot_content_max_size = get_config()["snapshot_content_max_size"] def _re_encode_content(mimetype, encoding, content_data): # encode textual content to utf-8 if needed - if mimetype.startswith('text/'): + if mimetype.startswith("text/"): # probably a malformed UTF-8 content, re-encode it # by replacing invalid chars with a substitution one - if encoding == 'unknown-8bit': - content_data = content_data.decode('utf-8', 'replace')\ - .encode('utf-8') - elif encoding not in ['utf-8', 'binary']: - content_data = content_data.decode(encoding, 'replace')\ - .encode('utf-8') - elif mimetype.startswith('application/octet-stream'): + if encoding == "unknown-8bit": + content_data = content_data.decode("utf-8", "replace").encode("utf-8") + elif encoding not in ["utf-8", "binary"]: + content_data = content_data.decode(encoding, "replace").encode("utf-8") + elif mimetype.startswith("application/octet-stream"): # file may detect a text content as binary # so try to decode it for display - encodings = ['us-ascii', 'utf-8'] - encodings += ['iso-8859-%s' % i for i in range(1, 17)] + encodings = ["us-ascii", "utf-8"] + encodings += ["iso-8859-%s" % i for i in range(1, 17)] for enc in encodings: try: - content_data = content_data.decode(enc).encode('utf-8') + content_data = content_data.decode(enc).encode("utf-8") except Exception as exc: sentry_sdk.capture_exception(exc) else: # ensure display in content view encoding = enc - mimetype = 'text/plain' + mimetype = "text/plain" break return mimetype, encoding, content_data -def request_content(query_string, max_size=content_display_max_size, - raise_if_unavailable=True, re_encode=True): +def request_content( + query_string, + max_size=content_display_max_size, + raise_if_unavailable=True, + re_encode=True, +): """Function that retrieves a content from the archive. Raw bytes content is first retrieved, then the content mime type. If the mime type is not stored in the archive, it will be computed using Python magic module. Args: query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value max_size: the maximum size for a content to retrieve (default to 1MB, no size limit if None) Returns: A tuple whose first member corresponds to the content raw bytes and second member the content mime type Raises: NotFoundExc if the content is not found """ content_data = service.lookup_content(query_string) filetype = None language = None license = None # requests to the indexer db may fail so properly handle # those cases in order to avoid content display errors try: filetype = service.lookup_content_filetype(query_string) language = service.lookup_content_language(query_string) license = service.lookup_content_license(query_string) except Exception as exc: sentry_sdk.capture_exception(exc) - mimetype = 'unknown' - encoding = 'unknown' + mimetype = "unknown" + encoding = "unknown" if filetype: - mimetype = filetype['mimetype'] - encoding = filetype['encoding'] + mimetype = filetype["mimetype"] + encoding = filetype["encoding"] # workaround when encountering corrupted data due to implicit # conversion from bytea to text in the indexer db (see T818) # TODO: Remove that code when all data have been correctly converted - if mimetype.startswith('\\'): + if mimetype.startswith("\\"): filetype = None - content_data['error_code'] = 200 - content_data['error_message'] = '' - content_data['error_description'] = '' + content_data["error_code"] = 200 + content_data["error_message"] = "" + content_data["error_description"] = "" - if not max_size or content_data['length'] < max_size: + if not max_size or content_data["length"] < max_size: try: content_raw = service.lookup_content_raw(query_string) except Exception as exc: if raise_if_unavailable: raise exc else: sentry_sdk.capture_exception(exc) - content_data['raw_data'] = None - content_data['error_code'] = 404 - content_data['error_description'] = \ - 'The bytes of the content are currently not available in the archive.' # noqa - content_data['error_message'] = \ - http_status_code_message[content_data['error_code']] + content_data["raw_data"] = None + content_data["error_code"] = 404 + content_data["error_description"] = ( + "The bytes of the content are currently not available " + "in the archive." + ) + content_data["error_message"] = http_status_code_message[ + content_data["error_code"] + ] else: - content_data['raw_data'] = content_raw['data'] + content_data["raw_data"] = content_raw["data"] if not filetype: - mimetype, encoding = \ - get_mimetype_and_encoding_for_content(content_data['raw_data']) # noqa + mimetype, encoding = get_mimetype_and_encoding_for_content( + content_data["raw_data"] + ) if re_encode: mimetype, encoding, raw_data = _re_encode_content( - mimetype, encoding, content_data['raw_data']) - content_data['raw_data'] = raw_data + mimetype, encoding, content_data["raw_data"] + ) + content_data["raw_data"] = raw_data else: - content_data['raw_data'] = None + content_data["raw_data"] = None - content_data['mimetype'] = mimetype - content_data['encoding'] = encoding + content_data["mimetype"] = mimetype + content_data["encoding"] = encoding if language: - content_data['language'] = language['lang'] + content_data["language"] = language["lang"] else: - content_data['language'] = 'not detected' + content_data["language"] = "not detected" if license: - content_data['licenses'] = ', '.join(license['facts'][0]['licenses']) + content_data["licenses"] = ", ".join(license["facts"][0]["licenses"]) else: - content_data['licenses'] = 'not detected' + content_data["licenses"] = "not detected" return content_data -_browsers_supported_image_mimes = set(['image/gif', 'image/png', - 'image/jpeg', 'image/bmp', - 'image/webp', 'image/svg', - 'image/svg+xml']) +_browsers_supported_image_mimes = set( + [ + "image/gif", + "image/png", + "image/jpeg", + "image/bmp", + "image/webp", + "image/svg", + "image/svg+xml", + ] +) def prepare_content_for_display(content_data, mime_type, path): """Function that prepares a content for HTML display. The function tries to associate a programming language to a content in order to perform syntax highlighting client-side using highlightjs. The language is determined using either the content filename or its mime type. If the mime type corresponds to an image format supported by web browsers, the content will be encoded in base64 for displaying the image. Args: content_data (bytes): raw bytes of the content mime_type (string): mime type of the content path (string): path of the content including filename Returns: A dict containing the content bytes (possibly different from the one provided as parameter if it is an image) under the key 'content_data and the corresponding highlightjs language class under the key 'language'. """ language = highlightjs.get_hljs_language_from_filename(path) if not language: language = highlightjs.get_hljs_language_from_mime_type(mime_type) if not language: - language = 'nohighlight' - elif mime_type.startswith('application/'): - mime_type = mime_type.replace('application/', 'text/') + language = "nohighlight" + elif mime_type.startswith("application/"): + mime_type = mime_type.replace("application/", "text/") - if mime_type.startswith('image/'): + if mime_type.startswith("image/"): if mime_type in _browsers_supported_image_mimes: - content_data = base64.b64encode(content_data).decode('ascii') + content_data = base64.b64encode(content_data).decode("ascii") else: content_data = None - if mime_type.startswith('image/svg'): - mime_type = 'image/svg+xml' + if mime_type.startswith("image/svg"): + mime_type = "image/svg+xml" - if mime_type.startswith('text/'): - content_data = content_data.decode('utf-8', errors='replace') + if mime_type.startswith("text/"): + content_data = content_data.decode("utf-8", errors="replace") - return {'content_data': content_data, - 'language': language, - 'mimetype': mime_type} + return {"content_data": content_data, "language": language, "mimetype": mime_type} def process_snapshot_branches(snapshot): """ Process a dictionary describing snapshot branches: extract those targeting revisions and releases, put them in two different lists, then sort those lists in lexicographical order of the branches' names. Args: snapshot_branches (dict): A dict describing the branches of a snapshot as returned for instance by :func:`swh.web.common.service.lookup_snapshot` Returns: tuple: A tuple whose first member is the sorted list of branches targeting revisions and second member the sorted list of branches targeting releases """ - snapshot_branches = snapshot['branches'] + snapshot_branches = snapshot["branches"] branches = {} branch_aliases = {} releases = {} revision_to_branch = defaultdict(set) revision_to_release = defaultdict(set) release_to_branch = defaultdict(set) for branch_name, target in snapshot_branches.items(): if not target: # FIXME: display branches with an unknown target anyway continue - target_id = target['target'] - target_type = target['target_type'] - if target_type == 'revision': + target_id = target["target"] + target_type = target["target_type"] + if target_type == "revision": branches[branch_name] = { - 'name': branch_name, - 'revision': target_id, + "name": branch_name, + "revision": target_id, } revision_to_branch[target_id].add(branch_name) - elif target_type == 'release': + elif target_type == "release": release_to_branch[target_id].add(branch_name) - elif target_type == 'alias': + elif target_type == "alias": branch_aliases[branch_name] = target_id # FIXME: handle pointers to other object types def _enrich_release_branch(branch, release): releases[branch] = { - 'name': release['name'], - 'branch_name': branch, - 'date': format_utc_iso_date(release['date']), - 'id': release['id'], - 'message': release['message'], - 'target_type': release['target_type'], - 'target': release['target'], + "name": release["name"], + "branch_name": branch, + "date": format_utc_iso_date(release["date"]), + "id": release["id"], + "message": release["message"], + "target_type": release["target_type"], + "target": release["target"], } def _enrich_revision_branch(branch, revision): - branches[branch].update({ - 'revision': revision['id'], - 'directory': revision['directory'], - 'date': format_utc_iso_date(revision['date']), - 'message': revision['message'] - }) - - releases_info = service.lookup_release_multiple( - release_to_branch.keys() - ) + branches[branch].update( + { + "revision": revision["id"], + "directory": revision["directory"], + "date": format_utc_iso_date(revision["date"]), + "message": revision["message"], + } + ) + + releases_info = service.lookup_release_multiple(release_to_branch.keys()) for release in releases_info: - branches_to_update = release_to_branch[release['id']] + branches_to_update = release_to_branch[release["id"]] for branch in branches_to_update: _enrich_release_branch(branch, release) - if release['target_type'] == 'revision': - revision_to_release[release['target']].update( - branches_to_update - ) + if release["target_type"] == "revision": + revision_to_release[release["target"]].update(branches_to_update) revisions = service.lookup_revision_multiple( set(revision_to_branch.keys()) | set(revision_to_release.keys()) ) for revision in revisions: if not revision: continue - for branch in revision_to_branch[revision['id']]: + for branch in revision_to_branch[revision["id"]]: _enrich_revision_branch(branch, revision) - for release in revision_to_release[revision['id']]: - releases[release]['directory'] = revision['directory'] + for release in revision_to_release[revision["id"]]: + releases[release]["directory"] = revision["directory"] for branch_alias, branch_target in branch_aliases.items(): if branch_target in branches: branches[branch_alias] = dict(branches[branch_target]) else: - snp = service.lookup_snapshot(snapshot['id'], - branches_from=branch_target, - branches_count=1) - if snp and branch_target in snp['branches']: + snp = service.lookup_snapshot( + snapshot["id"], branches_from=branch_target, branches_count=1 + ) + if snp and branch_target in snp["branches"]: - if snp['branches'][branch_target] is None: + if snp["branches"][branch_target] is None: continue - target_type = snp['branches'][branch_target]['target_type'] - target = snp['branches'][branch_target]['target'] - if target_type == 'revision': - branches[branch_alias] = snp['branches'][branch_target] + target_type = snp["branches"][branch_target]["target_type"] + target = snp["branches"][branch_target]["target"] + if target_type == "revision": + branches[branch_alias] = snp["branches"][branch_target] revision = service.lookup_revision(target) _enrich_revision_branch(branch_alias, revision) - elif target_type == 'release': + elif target_type == "release": release = service.lookup_release(target) _enrich_release_branch(branch_alias, release) if branch_alias in branches: - branches[branch_alias]['name'] = branch_alias + branches[branch_alias]["name"] = branch_alias - ret_branches = list(sorted(branches.values(), key=lambda b: b['name'])) - ret_releases = list(sorted(releases.values(), key=lambda b: b['name'])) + ret_branches = list(sorted(branches.values(), key=lambda b: b["name"])) + ret_releases = list(sorted(releases.values(), key=lambda b: b["name"])) return ret_branches, ret_releases def get_snapshot_content(snapshot_id): """Returns the lists of branches and releases associated to a swh snapshot. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: snapshot_id (str): hexadecimal representation of the snapshot identifier Returns: A tuple with two members. The first one is a list of dict describing the snapshot branches. The second one is a list of dict describing the snapshot releases. Raises: NotFoundExc if the snapshot does not exist """ - cache_entry_id = 'swh_snapshot_%s' % snapshot_id + cache_entry_id = "swh_snapshot_%s" % snapshot_id cache_entry = cache.get(cache_entry_id) if cache_entry: - return cache_entry['branches'], cache_entry['releases'] + return cache_entry["branches"], cache_entry["releases"] branches = [] releases = [] if snapshot_id: snapshot = service.lookup_snapshot( - snapshot_id, branches_count=snapshot_content_max_size) + snapshot_id, branches_count=snapshot_content_max_size + ) branches, releases = process_snapshot_branches(snapshot) - cache.set(cache_entry_id, { - 'branches': branches, - 'releases': releases, - }) + cache.set(cache_entry_id, {"branches": branches, "releases": releases,}) return branches, releases -def get_origin_visit_snapshot(origin_info, visit_ts=None, visit_id=None, - snapshot_id=None): +def get_origin_visit_snapshot( + origin_info, visit_ts=None, visit_id=None, snapshot_id=None +): """Returns the lists of branches and releases associated to a swh origin for a given visit. The visit is expressed by a timestamp. In the latter case, the closest visit from the provided timestamp will be used. If no visit parameter is provided, it returns the list of branches found for the latest visit. That list is put in cache in order to speedup the navigation in the swh-web/browse ui. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: origin_info (dict): a dict filled with origin information (id, url, type) visit_ts (int or str): an ISO date string or Unix timestamp to parse visit_id (int): optional visit id for disambiguation in case several visits have the same timestamp Returns: A tuple with two members. The first one is a list of dict describing the origin branches for the given visit. The second one is a list of dict describing the origin releases for the given visit. Raises: NotFoundExc if the origin or its visit are not found """ visit_info = get_origin_visit(origin_info, visit_ts, visit_id, snapshot_id) - return get_snapshot_content(visit_info['snapshot']) + return get_snapshot_content(visit_info["snapshot"]) def gen_link(url, link_text=None, link_attrs=None): """ Utility function for generating an HTML link to insert in Django templates. Args: url (str): an url link_text (str): optional text for the produced link, if not provided the url will be used link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ - attrs = ' ' + attrs = " " if link_attrs: for k, v in link_attrs.items(): attrs += '%s="%s" ' % (k, v) if not link_text: link_text = url - link = '%s' \ - % (attrs, escape(url), escape(link_text)) + link = '%s' % (attrs, escape(url), escape(link_text)) return mark_safe(link) def _snapshot_context_query_params(snapshot_context): query_params = None - if snapshot_context and snapshot_context['origin_info']: - origin_info = snapshot_context['origin_info'] - query_params = {'origin': origin_info['url']} - if 'timestamp' in snapshot_context['url_args']: - query_params['timestamp'] = \ - snapshot_context['url_args']['timestamp'] - if 'visit_id' in snapshot_context['query_params']: - query_params['visit_id'] = \ - snapshot_context['query_params']['visit_id'] + if snapshot_context and snapshot_context["origin_info"]: + origin_info = snapshot_context["origin_info"] + query_params = {"origin": origin_info["url"]} + if "timestamp" in snapshot_context["url_args"]: + query_params["timestamp"] = snapshot_context["url_args"]["timestamp"] + if "visit_id" in snapshot_context["query_params"]: + query_params["visit_id"] = snapshot_context["query_params"]["visit_id"] elif snapshot_context: - query_params = {'snapshot_id': snapshot_context['snapshot_id']} + query_params = {"snapshot_id": snapshot_context["snapshot_id"]} return query_params def gen_revision_url(revision_id, snapshot_context=None): """ Utility function for generating an url to a revision. Args: revision_id (str): a revision id snapshot_context (dict): if provided, generate snapshot-dependent browsing url Returns: str: The url to browse the revision """ query_params = _snapshot_context_query_params(snapshot_context) - return reverse('browse-revision', - url_args={'sha1_git': revision_id}, - query_params=query_params) + return reverse( + "browse-revision", url_args={"sha1_git": revision_id}, query_params=query_params + ) -def gen_revision_link(revision_id, shorten_id=False, snapshot_context=None, - link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_revision_link( + revision_id, + shorten_id=False, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a revision HTML view to insert in Django templates. Args: revision_id (str): a revision id shorten_id (boolean): whether to shorten the revision id to 7 characters for the link text snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: str: An HTML link in the form 'revision_id' """ if not revision_id: return None revision_url = gen_revision_url(revision_id, snapshot_context) if shorten_id: return gen_link(revision_url, revision_id[:7], link_attrs) else: if not link_text: link_text = revision_id return gen_link(revision_url, link_text, link_attrs) -def gen_directory_link(sha1_git, snapshot_context=None, link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_directory_link( + sha1_git, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a directory HTML view to insert in Django templates. Args: sha1_git (str): directory identifier link_text (str): optional text for the generated link (the directory id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) - directory_url = reverse('browse-directory', - url_args={'sha1_git': sha1_git}, - query_params=query_params) + directory_url = reverse( + "browse-directory", url_args={"sha1_git": sha1_git}, query_params=query_params + ) if not link_text: link_text = sha1_git return gen_link(directory_url, link_text, link_attrs) -def gen_snapshot_link(snapshot_id, snapshot_context=None, link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_snapshot_link( + snapshot_id, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a snapshot HTML view to insert in Django templates. Args: snapshot_id (str): snapshot identifier link_text (str): optional text for the generated link (the snapshot id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) - snapshot_url = reverse('browse-snapshot', - url_args={'snapshot_id': snapshot_id}, - query_params=query_params) + snapshot_url = reverse( + "browse-snapshot", + url_args={"snapshot_id": snapshot_id}, + query_params=query_params, + ) if not link_text: link_text = snapshot_id return gen_link(snapshot_url, link_text, link_attrs) -def gen_content_link(sha1_git, snapshot_context=None, link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_content_link( + sha1_git, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a content HTML view to insert in Django templates. Args: sha1_git (str): content identifier link_text (str): optional text for the generated link (the content sha1_git will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not sha1_git: return None query_params = _snapshot_context_query_params(snapshot_context) - content_url = reverse('browse-content', - url_args={'query_string': 'sha1_git:' + sha1_git}, - query_params=query_params) + content_url = reverse( + "browse-content", + url_args={"query_string": "sha1_git:" + sha1_git}, + query_params=query_params, + ) if not link_text: link_text = sha1_git return gen_link(content_url, link_text, link_attrs) def get_revision_log_url(revision_id, snapshot_context=None): """ Utility function for getting the URL for a revision log HTML view (possibly in the context of an origin). Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link Returns: The revision log view URL """ - query_params = {'revision': revision_id} - if snapshot_context and snapshot_context['origin_info']: - origin_info = snapshot_context['origin_info'] - url_args = {'origin_url': origin_info['url']} - if 'timestamp' in snapshot_context['url_args']: - url_args['timestamp'] = \ - snapshot_context['url_args']['timestamp'] - if 'visit_id' in snapshot_context['query_params']: - query_params['visit_id'] = \ - snapshot_context['query_params']['visit_id'] - revision_log_url = reverse('browse-origin-log', - url_args=url_args, - query_params=query_params) + query_params = {"revision": revision_id} + if snapshot_context and snapshot_context["origin_info"]: + origin_info = snapshot_context["origin_info"] + url_args = {"origin_url": origin_info["url"]} + if "timestamp" in snapshot_context["url_args"]: + url_args["timestamp"] = snapshot_context["url_args"]["timestamp"] + if "visit_id" in snapshot_context["query_params"]: + query_params["visit_id"] = snapshot_context["query_params"]["visit_id"] + revision_log_url = reverse( + "browse-origin-log", url_args=url_args, query_params=query_params + ) elif snapshot_context: - url_args = {'snapshot_id': snapshot_context['snapshot_id']} - revision_log_url = reverse('browse-snapshot-log', - url_args=url_args, - query_params=query_params) + url_args = {"snapshot_id": snapshot_context["snapshot_id"]} + revision_log_url = reverse( + "browse-snapshot-log", url_args=url_args, query_params=query_params + ) else: - revision_log_url = reverse('browse-revision-log', - url_args={'sha1_git': revision_id}) + revision_log_url = reverse( + "browse-revision-log", url_args={"sha1_git": revision_id} + ) return revision_log_url -def gen_revision_log_link(revision_id, snapshot_context=None, - link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_revision_log_link( + revision_id, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a revision log HTML view (possibly in the context of an origin) to insert in Django templates. Args: revision_id (str): revision identifier the history heads to snapshot_context (dict): if provided, generate snapshot-dependent browsing link link_text (str): optional text to use for the generated link (the revision id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ if not revision_id: return None revision_log_url = get_revision_log_url(revision_id, snapshot_context) if not link_text: link_text = revision_id return gen_link(revision_log_url, link_text, link_attrs) def gen_person_mail_link(person, link_text=None): """ Utility function for generating a mail link to a person to insert in Django templates. Args: person (dict): dictionary containing person data (*name*, *email*, *fullname*) link_text (str): optional text to use for the generated mail link (the person name will be used by default) Returns: str: A mail link to the person or the person name if no email is present in person data """ - person_name = person['name'] or person['fullname'] or 'None' + person_name = person["name"] or person["fullname"] or "None" if link_text is None: link_text = person_name - person_email = person['email'] if person['email'] else None - if person_email is None and '@' in person_name and ' ' not in person_name: + person_email = person["email"] if person["email"] else None + if person_email is None and "@" in person_name and " " not in person_name: person_email = person_name if person_email: - return gen_link(url='mailto:%s' % person_email, - link_text=link_text) + return gen_link(url="mailto:%s" % person_email, link_text=link_text) else: return person_name -def gen_release_link(sha1_git, snapshot_context=None, link_text='Browse', - link_attrs={'class': 'btn btn-default btn-sm', - 'role': 'button'}): +def gen_release_link( + sha1_git, + snapshot_context=None, + link_text="Browse", + link_attrs={"class": "btn btn-default btn-sm", "role": "button"}, +): """ Utility function for generating a link to a release HTML view to insert in Django templates. Args: sha1_git (str): release identifier link_text (str): optional text for the generated link (the release id will be used by default) link_attrs (dict): optional attributes (e.g. class) to add to the link Returns: An HTML link in the form 'link_text' """ query_params = _snapshot_context_query_params(snapshot_context) - release_url = reverse('browse-release', - url_args={'sha1_git': sha1_git}, - query_params=query_params) + release_url = reverse( + "browse-release", url_args={"sha1_git": sha1_git}, query_params=query_params + ) if not link_text: link_text = sha1_git return gen_link(release_url, link_text, link_attrs) def format_log_entries(revision_log, per_page, snapshot_context=None): """ Utility functions that process raw revision log data for HTML display. Its purpose is to: * add links to relevant browse views * format date in human readable format * truncate the message log Args: revision_log (list): raw revision log as returned by the swh-web api per_page (int): number of log entries per page snapshot_context (dict): if provided, generate snapshot-dependent browsing link """ revision_log_data = [] for i, rev in enumerate(revision_log): if i == per_page: break - author_name = 'None' - author_fullname = 'None' - committer_fullname = 'None' - if rev['author']: - author_name = gen_person_mail_link(rev['author']) - author_fullname = rev['author']['fullname'] - if rev['committer']: - committer_fullname = rev['committer']['fullname'] - author_date = format_utc_iso_date(rev['date']) - committer_date = format_utc_iso_date(rev['committer_date']) - - tooltip = 'revision %s\n' % rev['id'] - tooltip += 'author: %s\n' % author_fullname - tooltip += 'author date: %s\n' % author_date - tooltip += 'committer: %s\n' % committer_fullname - tooltip += 'committer date: %s\n\n' % committer_date - if rev['message']: - tooltip += textwrap.indent(rev['message'], ' '*4) - - revision_log_data.append({ - 'author': author_name, - 'id': rev['id'][:7], - 'message': rev['message'], - 'date': author_date, - 'commit_date': committer_date, - 'url': gen_revision_url(rev['id'], snapshot_context), - 'tooltip': tooltip - }) + author_name = "None" + author_fullname = "None" + committer_fullname = "None" + if rev["author"]: + author_name = gen_person_mail_link(rev["author"]) + author_fullname = rev["author"]["fullname"] + if rev["committer"]: + committer_fullname = rev["committer"]["fullname"] + author_date = format_utc_iso_date(rev["date"]) + committer_date = format_utc_iso_date(rev["committer_date"]) + + tooltip = "revision %s\n" % rev["id"] + tooltip += "author: %s\n" % author_fullname + tooltip += "author date: %s\n" % author_date + tooltip += "committer: %s\n" % committer_fullname + tooltip += "committer date: %s\n\n" % committer_date + if rev["message"]: + tooltip += textwrap.indent(rev["message"], " " * 4) + + revision_log_data.append( + { + "author": author_name, + "id": rev["id"][:7], + "message": rev["message"], + "date": author_date, + "commit_date": committer_date, + "url": gen_revision_url(rev["id"], snapshot_context), + "tooltip": tooltip, + } + ) return revision_log_data -def get_snapshot_context(snapshot_id=None, origin_url=None, - timestamp=None, visit_id=None): +def get_snapshot_context( + snapshot_id=None, origin_url=None, timestamp=None, visit_id=None +): """ Utility function to compute relevant information when navigating the archive in a snapshot context. The snapshot is either referenced by its id or it will be retrieved from an origin visit. Args: snapshot_id (str): hexadecimal representation of a snapshot identifier, all other parameters will be ignored if it is provided origin_url (str): the origin_url (e.g. https://github.com/(user)/(repo)/) timestamp (str): a datetime string for retrieving the closest visit of the origin visit_id (int): optional visit id for disambiguation in case of several visits with the same timestamp Returns: A dict with the following entries: * origin_info: dict containing origin information * visit_info: dict containing visit information * branches: the list of branches for the origin found during the visit * releases: the list of releases for the origin found during the visit * origin_browse_url: the url to browse the origin * origin_branches_url: the url to browse the origin branches * origin_releases_url': the url to browse the origin releases * origin_visit_url: the url to browse the snapshot of the origin found during the visit * url_args: dict containing url arguments to use when browsing in the context of the origin and its visit Raises: swh.web.common.exc.NotFoundExc: if no snapshot is found for the visit of an origin. """ origin_info = None visit_info = None url_args = None query_params = {} branches = [] releases = [] browse_url = None visit_url = None branches_url = None releases_url = None - swh_type = 'snapshot' + swh_type = "snapshot" if origin_url: - swh_type = 'origin' - origin_info = service.lookup_origin({'url': origin_url}) + swh_type = "origin" + origin_info = service.lookup_origin({"url": origin_url}) - visit_info = get_origin_visit(origin_info, timestamp, visit_id, - snapshot_id) - fmt_date = format_utc_iso_date(visit_info['date']) - visit_info['fmt_date'] = fmt_date - snapshot_id = visit_info['snapshot'] + visit_info = get_origin_visit(origin_info, timestamp, visit_id, snapshot_id) + fmt_date = format_utc_iso_date(visit_info["date"]) + visit_info["fmt_date"] = fmt_date + snapshot_id = visit_info["snapshot"] if not snapshot_id: - raise NotFoundExc('No snapshot associated to the visit of origin ' - '%s on %s' % (escape(origin_url), fmt_date)) + raise NotFoundExc( + "No snapshot associated to the visit of origin " + "%s on %s" % (escape(origin_url), fmt_date) + ) # provided timestamp is not necessarily equals to the one # of the retrieved visit, so get the exact one in order # use it in the urls generated below if timestamp: - timestamp = visit_info['date'] + timestamp = visit_info["date"] - branches, releases = \ - get_origin_visit_snapshot(origin_info, timestamp, visit_id, - snapshot_id) + branches, releases = get_origin_visit_snapshot( + origin_info, timestamp, visit_id, snapshot_id + ) - url_args = {'origin_url': origin_info['url']} + url_args = {"origin_url": origin_info["url"]} - query_params = {'visit_id': visit_id} + query_params = {"visit_id": visit_id} - browse_url = reverse('browse-origin-visits', - url_args=url_args) + browse_url = reverse("browse-origin-visits", url_args=url_args) if timestamp: - url_args['timestamp'] = format_utc_iso_date(timestamp, - '%Y-%m-%dT%H:%M:%S') - visit_url = reverse('browse-origin-directory', - url_args=url_args, - query_params=query_params) - visit_info['url'] = visit_url - - branches_url = reverse('browse-origin-branches', - url_args=url_args, - query_params=query_params) - - releases_url = reverse('browse-origin-releases', - url_args=url_args, - query_params=query_params) + url_args["timestamp"] = format_utc_iso_date(timestamp, "%Y-%m-%dT%H:%M:%S") + visit_url = reverse( + "browse-origin-directory", url_args=url_args, query_params=query_params + ) + visit_info["url"] = visit_url + + branches_url = reverse( + "browse-origin-branches", url_args=url_args, query_params=query_params + ) + + releases_url = reverse( + "browse-origin-releases", url_args=url_args, query_params=query_params + ) elif snapshot_id: branches, releases = get_snapshot_content(snapshot_id) - url_args = {'snapshot_id': snapshot_id} - browse_url = reverse('browse-snapshot', - url_args=url_args) - branches_url = reverse('browse-snapshot-branches', - url_args=url_args) + url_args = {"snapshot_id": snapshot_id} + browse_url = reverse("browse-snapshot", url_args=url_args) + branches_url = reverse("browse-snapshot-branches", url_args=url_args) - releases_url = reverse('browse-snapshot-releases', - url_args=url_args) + releases_url = reverse("browse-snapshot-releases", url_args=url_args) releases = list(reversed(releases)) snapshot_sizes = service.lookup_snapshot_sizes(snapshot_id) is_empty = sum(snapshot_sizes.values()) == 0 - swh_snp_id = persistent_identifier('snapshot', snapshot_id) + swh_snp_id = persistent_identifier("snapshot", snapshot_id) return { - 'swh_type': swh_type, - 'swh_object_id': swh_snp_id, - 'snapshot_id': snapshot_id, - 'snapshot_sizes': snapshot_sizes, - 'is_empty': is_empty, - 'origin_info': origin_info, - 'visit_info': visit_info, - 'branches': branches, - 'releases': releases, - 'branch': None, - 'release': None, - 'browse_url': browse_url, - 'branches_url': branches_url, - 'releases_url': releases_url, - 'url_args': url_args, - 'query_params': query_params + "swh_type": swh_type, + "swh_object_id": swh_snp_id, + "snapshot_id": snapshot_id, + "snapshot_sizes": snapshot_sizes, + "is_empty": is_empty, + "origin_info": origin_info, + "visit_info": visit_info, + "branches": branches, + "releases": releases, + "branch": None, + "release": None, + "browse_url": browse_url, + "branches_url": branches_url, + "releases_url": releases_url, + "url_args": url_args, + "query_params": query_params, } # list of common readme names ordered by preference # (lower indices have higher priority) _common_readme_names = [ "readme.markdown", "readme.md", "readme.rst", "readme.txt", - "readme" + "readme", ] def get_readme_to_display(readmes): """ Process a list of readme files found in a directory in order to find the adequate one to display. Args: readmes: a list of dict where keys are readme file names and values are readme sha1s Returns: A tuple (readme_name, readme_sha1) """ readme_name = None readme_url = None readme_sha1 = None readme_html = None - lc_readmes = {k.lower(): {'orig_name': k, 'sha1': v} - for k, v in readmes.items()} + lc_readmes = {k.lower(): {"orig_name": k, "sha1": v} for k, v in readmes.items()} # look for readme names according to the preference order # defined by the _common_readme_names list for common_readme_name in _common_readme_names: if common_readme_name in lc_readmes: - readme_name = lc_readmes[common_readme_name]['orig_name'] - readme_sha1 = lc_readmes[common_readme_name]['sha1'] - readme_url = reverse('browse-content-raw', - url_args={'query_string': readme_sha1}, - query_params={'re_encode': 'true'}) + readme_name = lc_readmes[common_readme_name]["orig_name"] + readme_sha1 = lc_readmes[common_readme_name]["sha1"] + readme_url = reverse( + "browse-content-raw", + url_args={"query_string": readme_sha1}, + query_params={"re_encode": "true"}, + ) break # otherwise pick the first readme like file if any if not readme_name and len(readmes.items()) > 0: readme_name = next(iter(readmes)) readme_sha1 = readmes[readme_name] - readme_url = reverse('browse-content-raw', - url_args={'query_string': readme_sha1}, - query_params={'re_encode': 'true'}) + readme_url = reverse( + "browse-content-raw", + url_args={"query_string": readme_sha1}, + query_params={"re_encode": "true"}, + ) # convert rst README to html server side as there is # no viable solution to perform that task client side - if readme_name and readme_name.endswith('.rst'): - cache_entry_id = 'readme_%s' % readme_sha1 + if readme_name and readme_name.endswith(".rst"): + cache_entry_id = "readme_%s" % readme_sha1 cache_entry = cache.get(cache_entry_id) if cache_entry: readme_html = cache_entry else: try: rst_doc = request_content(readme_sha1) - readme_html = rst_to_html(rst_doc['raw_data']) + readme_html = rst_to_html(rst_doc["raw_data"]) cache.set(cache_entry_id, readme_html) except Exception as exc: sentry_sdk.capture_exception(exc) - readme_html = 'Readme bytes are not available' + readme_html = "Readme bytes are not available" return readme_name, readme_url, readme_html def get_swh_persistent_ids(swh_objects, snapshot_context=None): """ Returns a list of dict containing info related to persistent identifiers of swh objects. Args: swh_objects (list): a list of dict with the following keys: * type: swh object type (content/directory/release/revision/snapshot) * id: swh object id snapshot_context (dict): optional parameter describing the snapshot in which the object has been found Returns: list: a list of dict with the following keys: * object_type: the swh object type (content/directory/release/revision/snapshot) * object_icon: the swh object icon to use in HTML views * swh_id: the computed swh object persistent identifier * swh_id_url: the url resolving the persistent identifier * show_options: boolean indicating if the persistent id options must be displayed in persistent ids HTML view """ swh_ids = [] for swh_object in swh_objects: - if not swh_object['id']: + if not swh_object["id"]: continue - swh_id = get_swh_persistent_id(swh_object['type'], swh_object['id']) - show_options = swh_object['type'] == 'content' or \ - (snapshot_context and snapshot_context['origin_info'] is not None) - - object_icon = swh_object_icons[swh_object['type']] - - swh_ids.append({ - 'object_type': swh_object['type'], - 'object_id': swh_object['id'], - 'object_icon': object_icon, - 'swh_id': swh_id, - 'swh_id_url': reverse('browse-swh-id', - url_args={'swh_id': swh_id}), - 'show_options': show_options - }) + swh_id = get_swh_persistent_id(swh_object["type"], swh_object["id"]) + show_options = swh_object["type"] == "content" or ( + snapshot_context and snapshot_context["origin_info"] is not None + ) + + object_icon = swh_object_icons[swh_object["type"]] + + swh_ids.append( + { + "object_type": swh_object["type"], + "object_id": swh_object["id"], + "object_icon": object_icon, + "swh_id": swh_id, + "swh_id_url": reverse("browse-swh-id", url_args={"swh_id": swh_id}), + "show_options": show_options, + } + ) return swh_ids diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py index 30c15537..191e2d31 100644 --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -1,334 +1,354 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import difflib import json from distutils.util import strtobool from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat import sentry_sdk from swh.model.hashutil import hash_to_hex from swh.web.common import query, service, highlightjs -from swh.web.common.utils import ( - reverse, gen_path_info, swh_object_icons -) +from swh.web.common.utils import reverse, gen_path_info, swh_object_icons from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.utils import ( - request_content, prepare_content_for_display, - content_display_max_size, get_snapshot_context, - get_swh_persistent_ids, gen_link, gen_directory_link + request_content, + prepare_content_for_display, + content_display_max_size, + get_snapshot_context, + get_swh_persistent_ids, + gen_link, + gen_directory_link, ) from swh.web.browse.browseurls import browse_route -@browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/raw/', - view_name='browse-content-raw', - checksum_args=['query_string']) +@browse_route( + r"content/(?P[0-9a-z_:]*[0-9a-f]+.)/raw/", + view_name="browse-content-raw", + checksum_args=["query_string"], +) def content_raw(request, query_string): """Django view that produces a raw display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/raw/` """ try: - re_encode = bool(strtobool(request.GET.get('re_encode', 'false'))) + re_encode = bool(strtobool(request.GET.get("re_encode", "false"))) algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) - content_data = request_content(query_string, max_size=None, - re_encode=re_encode) + content_data = request_content(query_string, max_size=None, re_encode=re_encode) except Exception as exc: return handle_view_exception(request, exc) - filename = request.GET.get('filename', None) + filename = request.GET.get("filename", None) if not filename: - filename = '%s_%s' % (algo, checksum) - - if content_data['mimetype'].startswith('text/') or \ - content_data['mimetype'] == 'inode/x-empty': - response = HttpResponse(content_data['raw_data'], - content_type="text/plain") - response['Content-disposition'] = 'filename=%s' % filename + filename = "%s_%s" % (algo, checksum) + + if ( + content_data["mimetype"].startswith("text/") + or content_data["mimetype"] == "inode/x-empty" + ): + response = HttpResponse(content_data["raw_data"], content_type="text/plain") + response["Content-disposition"] = "filename=%s" % filename else: - response = HttpResponse(content_data['raw_data'], - content_type='application/octet-stream') - response['Content-disposition'] = 'attachment; filename=%s' % filename + response = HttpResponse( + content_data["raw_data"], content_type="application/octet-stream" + ) + response["Content-disposition"] = "attachment; filename=%s" % filename return response _auto_diff_size_limit = 20000 -@browse_route(r'content/(?P.*)/diff/(?P.*)', # noqa - view_name='diff-contents') +@browse_route( + r"content/(?P.*)/diff/(?P.*)", + view_name="diff-contents", +) def _contents_diff(request, from_query_string, to_query_string): """ Browse endpoint used to compute unified diffs between two contents. Diffs are generated only if the two contents are textual. By default, diffs whose size are greater than 20 kB will not be generated. To force the generation of large diffs, the 'force' boolean query parameter must be used. Args: request: input django http request from_query_string: a string of the form "[ALGO_HASH:]HASH" where optional ALGO_HASH can be either ``sha1``, ``sha1_git``, ``sha256``, or ``blake2s256`` (default to ``sha1``) and HASH the hexadecimal representation of the hash value identifying the first content to_query_string: same as above for identifying the second content Returns: A JSON object containing the unified diff. """ diff_data = {} content_from = None content_to = None content_from_size = 0 content_to_size = 0 content_from_lines = [] content_to_lines = [] - force = request.GET.get('force', 'false') - path = request.GET.get('path', None) - language = 'nohighlight' + force = request.GET.get("force", "false") + path = request.GET.get("path", None) + language = "nohighlight" force = bool(strtobool(force)) if from_query_string == to_query_string: - diff_str = 'File renamed without changes' + diff_str = "File renamed without changes" else: try: text_diff = True if from_query_string: - content_from = \ - request_content(from_query_string, max_size=None) - content_from_display_data = \ - prepare_content_for_display(content_from['raw_data'], - content_from['mimetype'], path) - language = content_from_display_data['language'] - content_from_size = content_from['length'] - if not (content_from['mimetype'].startswith('text/') or - content_from['mimetype'] == 'inode/x-empty'): + content_from = request_content(from_query_string, max_size=None) + content_from_display_data = prepare_content_for_display( + content_from["raw_data"], content_from["mimetype"], path + ) + language = content_from_display_data["language"] + content_from_size = content_from["length"] + if not ( + content_from["mimetype"].startswith("text/") + or content_from["mimetype"] == "inode/x-empty" + ): text_diff = False if text_diff and to_query_string: content_to = request_content(to_query_string, max_size=None) content_to_display_data = prepare_content_for_display( - content_to['raw_data'], content_to['mimetype'], path) - language = content_to_display_data['language'] - content_to_size = content_to['length'] - if not (content_to['mimetype'].startswith('text/') or - content_to['mimetype'] == 'inode/x-empty'): + content_to["raw_data"], content_to["mimetype"], path + ) + language = content_to_display_data["language"] + content_to_size = content_to["length"] + if not ( + content_to["mimetype"].startswith("text/") + or content_to["mimetype"] == "inode/x-empty" + ): text_diff = False diff_size = abs(content_to_size - content_from_size) if not text_diff: - diff_str = 'Diffs are not generated for non textual content' - language = 'nohighlight' + diff_str = "Diffs are not generated for non textual content" + language = "nohighlight" elif not force and diff_size > _auto_diff_size_limit: - diff_str = 'Large diffs are not automatically computed' - language = 'nohighlight' + diff_str = "Large diffs are not automatically computed" + language = "nohighlight" else: if content_from: - content_from_lines = \ - content_from['raw_data'].decode('utf-8')\ - .splitlines(True) - if content_from_lines and \ - content_from_lines[-1][-1] != '\n': - content_from_lines[-1] += '[swh-no-nl-marker]\n' + content_from_lines = ( + content_from["raw_data"].decode("utf-8").splitlines(True) + ) + if content_from_lines and content_from_lines[-1][-1] != "\n": + content_from_lines[-1] += "[swh-no-nl-marker]\n" if content_to: - content_to_lines = content_to['raw_data'].decode('utf-8')\ - .splitlines(True) - if content_to_lines and content_to_lines[-1][-1] != '\n': - content_to_lines[-1] += '[swh-no-nl-marker]\n' - - diff_lines = difflib.unified_diff(content_from_lines, - content_to_lines) - diff_str = ''.join(list(diff_lines)[2:]) + content_to_lines = ( + content_to["raw_data"].decode("utf-8").splitlines(True) + ) + if content_to_lines and content_to_lines[-1][-1] != "\n": + content_to_lines[-1] += "[swh-no-nl-marker]\n" + + diff_lines = difflib.unified_diff(content_from_lines, content_to_lines) + diff_str = "".join(list(diff_lines)[2:]) except Exception as exc: sentry_sdk.capture_exception(exc) diff_str = str(exc) - diff_data['diff_str'] = diff_str - diff_data['language'] = language - diff_data_json = json.dumps(diff_data, separators=(',', ': ')) - return HttpResponse(diff_data_json, content_type='application/json') + diff_data["diff_str"] = diff_str + diff_data["language"] = language + diff_data_json = json.dumps(diff_data, separators=(",", ": ")) + return HttpResponse(diff_data_json, content_type="application/json") -@browse_route(r'content/(?P[0-9a-z_:]*[0-9a-f]+.)/', - view_name='browse-content', - checksum_args=['query_string']) +@browse_route( + r"content/(?P[0-9a-z_:]*[0-9a-f]+.)/", + view_name="browse-content", + checksum_args=["query_string"], +) def content_display(request, query_string): """Django view that produces an HTML display of a content identified by its hash value. The url that points to it is :http:get:`/browse/content/[(algo_hash):](hash)/` """ try: algo, checksum = query.parse_hash(query_string) checksum = hash_to_hex(checksum) - content_data = request_content(query_string, - raise_if_unavailable=False) - origin_url = request.GET.get('origin_url', None) - selected_language = request.GET.get('language', None) + content_data = request_content(query_string, raise_if_unavailable=False) + origin_url = request.GET.get("origin_url", None) + selected_language = request.GET.get("language", None) if not origin_url: - origin_url = request.GET.get('origin', None) + origin_url = request.GET.get("origin", None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(origin_url=origin_url) except NotFoundExc: - raw_cnt_url = reverse('browse-content', - url_args={'query_string': query_string}) - error_message = \ - ('The Software Heritage archive has a content ' - 'with the hash you provided but the origin ' - 'mentioned in your request appears broken: %s. ' - 'Please check the URL and try again.\n\n' - 'Nevertheless, you can still browse the content ' - 'without origin information: %s' - % (gen_link(origin_url), gen_link(raw_cnt_url))) + raw_cnt_url = reverse( + "browse-content", url_args={"query_string": query_string} + ) + error_message = ( + "The Software Heritage archive has a content " + "with the hash you provided but the origin " + "mentioned in your request appears broken: %s. " + "Please check the URL and try again.\n\n" + "Nevertheless, you can still browse the content " + "without origin information: %s" + % (gen_link(origin_url), gen_link(raw_cnt_url)) + ) raise NotFoundExc(error_message) if snapshot_context: - snapshot_context['visit_info'] = None + snapshot_context["visit_info"] = None except Exception as exc: return handle_view_exception(request, exc) - path = request.GET.get('path', None) + path = request.GET.get("path", None) content = None language = None mimetype = None - if content_data['raw_data'] is not None: + if content_data["raw_data"] is not None: content_display_data = prepare_content_for_display( - content_data['raw_data'], content_data['mimetype'], path) - content = content_display_data['content_data'] - language = content_display_data['language'] - mimetype = content_display_data['mimetype'] + content_data["raw_data"], content_data["mimetype"], path + ) + content = content_display_data["content_data"] + language = content_display_data["language"] + mimetype = content_display_data["mimetype"] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None - if mimetype and 'text/' in mimetype: + if mimetype and "text/" in mimetype: available_languages = highlightjs.get_supported_languages() root_dir = None filename = None path_info = None directory_id = None directory_url = None - query_params = {'origin': origin_url} + query_params = {"origin": origin_url} breadcrumbs = [] if path: - split_path = path.split('/') + split_path = path.split("/") root_dir = split_path[0] filename = split_path[-1] if root_dir != path: - path = path.replace(root_dir + '/', '') - path = path[:-len(filename)] + path = path.replace(root_dir + "/", "") + path = path[: -len(filename)] path_info = gen_path_info(path) - dir_url = reverse('browse-directory', - url_args={'sha1_git': root_dir}, - query_params=query_params) - breadcrumbs.append({'name': root_dir[:7], - 'url': dir_url}) + dir_url = reverse( + "browse-directory", + url_args={"sha1_git": root_dir}, + query_params=query_params, + ) + breadcrumbs.append({"name": root_dir[:7], "url": dir_url}) for pi in path_info: - dir_url = reverse('browse-directory', - url_args={'sha1_git': root_dir, - 'path': pi['path']}, - query_params=query_params) - breadcrumbs.append({'name': pi['name'], - 'url': dir_url}) - breadcrumbs.append({'name': filename, - 'url': None}) + dir_url = reverse( + "browse-directory", + url_args={"sha1_git": root_dir, "path": pi["path"]}, + query_params=query_params, + ) + breadcrumbs.append({"name": pi["name"], "url": dir_url}) + breadcrumbs.append({"name": filename, "url": None}) if path and root_dir != path: try: dir_info = service.lookup_directory_with_path(root_dir, path) - directory_id = dir_info['target'] + directory_id = dir_info["target"] except Exception as exc: return handle_view_exception(request, exc) elif root_dir != path: directory_id = root_dir if directory_id: directory_url = gen_directory_link(directory_id) - query_params = {'filename': filename} + query_params = {"filename": filename} - content_raw_url = reverse('browse-content-raw', - url_args={'query_string': query_string}, - query_params=query_params) + content_raw_url = reverse( + "browse-content-raw", + url_args={"query_string": query_string}, + query_params=query_params, + ) content_metadata = { - 'sha1': content_data['checksums']['sha1'], - 'sha1_git': content_data['checksums']['sha1_git'], - 'sha256': content_data['checksums']['sha256'], - 'blake2s256': content_data['checksums']['blake2s256'], - 'mimetype': content_data['mimetype'], - 'encoding': content_data['encoding'], - 'size': filesizeformat(content_data['length']), - 'language': content_data['language'], - 'licenses': content_data['licenses'], - 'filename': filename, - 'directory': directory_id, - 'context-independent directory': directory_url + "sha1": content_data["checksums"]["sha1"], + "sha1_git": content_data["checksums"]["sha1_git"], + "sha256": content_data["checksums"]["sha256"], + "blake2s256": content_data["checksums"]["blake2s256"], + "mimetype": content_data["mimetype"], + "encoding": content_data["encoding"], + "size": filesizeformat(content_data["length"]), + "language": content_data["language"], + "licenses": content_data["licenses"], + "filename": filename, + "directory": directory_id, + "context-independent directory": directory_url, } if filename: - content_metadata['filename'] = filename + content_metadata["filename"] = filename - sha1_git = content_data['checksums']['sha1_git'] - swh_ids = get_swh_persistent_ids([{'type': 'content', - 'id': sha1_git}]) + sha1_git = content_data["checksums"]["sha1_git"] + swh_ids = get_swh_persistent_ids([{"type": "content", "id": sha1_git}]) - heading = 'Content - %s' % sha1_git + heading = "Content - %s" % sha1_git if breadcrumbs: - content_path = '/'.join([bc['name'] for bc in breadcrumbs]) - heading += ' - %s' % content_path - - return render(request, 'browse/content.html', - {'heading': heading, - 'swh_object_id': swh_ids[0]['swh_id'], - 'swh_object_name': 'Content', - 'swh_object_metadata': content_metadata, - 'content': content, - 'content_size': content_data['length'], - 'max_content_size': content_display_max_size, - 'mimetype': mimetype, - 'language': language, - 'available_languages': available_languages, - 'breadcrumbs': breadcrumbs, - 'top_right_link': { - 'url': content_raw_url, - 'icon': swh_object_icons['content'], - 'text': 'Raw File' - }, - 'snapshot_context': snapshot_context, - 'vault_cooking': None, - 'show_actions_menu': True, - 'swh_ids': swh_ids, - 'error_code': content_data['error_code'], - 'error_message': content_data['error_message'], - 'error_description': content_data['error_description']}, - status=content_data['error_code']) + content_path = "/".join([bc["name"] for bc in breadcrumbs]) + heading += " - %s" % content_path + + return render( + request, + "browse/content.html", + { + "heading": heading, + "swh_object_id": swh_ids[0]["swh_id"], + "swh_object_name": "Content", + "swh_object_metadata": content_metadata, + "content": content, + "content_size": content_data["length"], + "max_content_size": content_display_max_size, + "mimetype": mimetype, + "language": language, + "available_languages": available_languages, + "breadcrumbs": breadcrumbs, + "top_right_link": { + "url": content_raw_url, + "icon": swh_object_icons["content"], + "text": "Raw File", + }, + "snapshot_context": snapshot_context, + "vault_cooking": None, + "show_actions_menu": True, + "swh_ids": swh_ids, + "error_code": content_data["error_code"], + "error_message": content_data["error_message"], + "error_description": content_data["error_description"], + }, + status=content_data["error_code"], + ) diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py index bd38d5bf..31e3caad 100644 --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -1,179 +1,211 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from django.http import HttpResponse from django.shortcuts import render, redirect from django.template.defaultfilters import filesizeformat import sentry_sdk from swh.web.common import service -from swh.web.common.utils import ( - reverse, gen_path_info -) +from swh.web.common.utils import reverse, gen_path_info from swh.web.common.exc import handle_view_exception, NotFoundExc from swh.web.browse.utils import ( - get_directory_entries, get_snapshot_context, - get_readme_to_display, get_swh_persistent_ids, - gen_link + get_directory_entries, + get_snapshot_context, + get_readme_to_display, + get_swh_persistent_ids, + gen_link, ) from swh.web.browse.browseurls import browse_route -@browse_route(r'directory/(?P[0-9a-f]+)/', - r'directory/(?P[0-9a-f]+)/(?P.+)/', - view_name='browse-directory', - checksum_args=['sha1_git']) +@browse_route( + r"directory/(?P[0-9a-f]+)/", + r"directory/(?P[0-9a-f]+)/(?P.+)/", + view_name="browse-directory", + checksum_args=["sha1_git"], +) def directory_browse(request, sha1_git, path=None): """Django view for browsing the content of a directory identified by its sha1_git value. The url that points to it is :http:get:`/browse/directory/(sha1_git)/[(path)/]` """ root_sha1_git = sha1_git try: if path: dir_info = service.lookup_directory_with_path(sha1_git, path) - sha1_git = dir_info['target'] + sha1_git = dir_info["target"] dirs, files = get_directory_entries(sha1_git) - origin_url = request.GET.get('origin_url', None) + origin_url = request.GET.get("origin_url", None) if not origin_url: - origin_url = request.GET.get('origin', None) + origin_url = request.GET.get("origin", None) snapshot_context = None if origin_url: try: snapshot_context = get_snapshot_context(origin_url=origin_url) except NotFoundExc: - raw_dir_url = reverse('browse-directory', - url_args={'sha1_git': sha1_git}) - error_message = \ - ('The Software Heritage archive has a directory ' - 'with the hash you provided but the origin ' - 'mentioned in your request appears broken: %s. ' - 'Please check the URL and try again.\n\n' - 'Nevertheless, you can still browse the directory ' - 'without origin information: %s' - % (gen_link(origin_url), gen_link(raw_dir_url))) + raw_dir_url = reverse( + "browse-directory", url_args={"sha1_git": sha1_git} + ) + error_message = ( + "The Software Heritage archive has a directory " + "with the hash you provided but the origin " + "mentioned in your request appears broken: %s. " + "Please check the URL and try again.\n\n" + "Nevertheless, you can still browse the directory " + "without origin information: %s" + % (gen_link(origin_url), gen_link(raw_dir_url)) + ) raise NotFoundExc(error_message) if snapshot_context: - snapshot_context['visit_info'] = None + snapshot_context["visit_info"] = None except Exception as exc: return handle_view_exception(request, exc) path_info = gen_path_info(path) - query_params = {'origin': origin_url} + query_params = {"origin": origin_url} breadcrumbs = [] - breadcrumbs.append({'name': root_sha1_git[:7], - 'url': reverse('browse-directory', - url_args={'sha1_git': root_sha1_git}, - query_params=query_params)}) + breadcrumbs.append( + { + "name": root_sha1_git[:7], + "url": reverse( + "browse-directory", + url_args={"sha1_git": root_sha1_git}, + query_params=query_params, + ), + } + ) for pi in path_info: - breadcrumbs.append({'name': pi['name'], - 'url': reverse('browse-directory', - url_args={'sha1_git': root_sha1_git, - 'path': pi['path']}, - query_params=query_params)}) - - path = '' if path is None else (path + '/') + breadcrumbs.append( + { + "name": pi["name"], + "url": reverse( + "browse-directory", + url_args={"sha1_git": root_sha1_git, "path": pi["path"]}, + query_params=query_params, + ), + } + ) + + path = "" if path is None else (path + "/") for d in dirs: - if d['type'] == 'rev': - d['url'] = reverse('browse-revision', - url_args={'sha1_git': d['target']}, - query_params=query_params) + if d["type"] == "rev": + d["url"] = reverse( + "browse-revision", + url_args={"sha1_git": d["target"]}, + query_params=query_params, + ) else: - d['url'] = reverse('browse-directory', - url_args={'sha1_git': root_sha1_git, - 'path': path + d['name']}, - query_params=query_params) + d["url"] = reverse( + "browse-directory", + url_args={"sha1_git": root_sha1_git, "path": path + d["name"]}, + query_params=query_params, + ) sum_file_sizes = 0 readmes = {} for f in files: - query_string = 'sha1_git:' + f['target'] - f['url'] = reverse('browse-content', - url_args={'query_string': query_string}, - query_params={'path': root_sha1_git + '/' + - path + f['name'], - 'origin': origin_url}) - if f['length'] is not None: - sum_file_sizes += f['length'] - f['length'] = filesizeformat(f['length']) - if f['name'].lower().startswith('readme'): - readmes[f['name']] = f['checksums']['sha1'] + query_string = "sha1_git:" + f["target"] + f["url"] = reverse( + "browse-content", + url_args={"query_string": query_string}, + query_params={ + "path": root_sha1_git + "/" + path + f["name"], + "origin": origin_url, + }, + ) + if f["length"] is not None: + sum_file_sizes += f["length"] + f["length"] = filesizeformat(f["length"]) + if f["name"].lower().startswith("readme"): + readmes[f["name"]] = f["checksums"]["sha1"] readme_name, readme_url, readme_html = get_readme_to_display(readmes) sum_file_sizes = filesizeformat(sum_file_sizes) - dir_metadata = {"directory": sha1_git, - "number of regular files": len(files), - "number of subdirectories": len(dirs), - "sum of regular file sizes": sum_file_sizes} + dir_metadata = { + "directory": sha1_git, + "number of regular files": len(files), + "number of subdirectories": len(dirs), + "sum of regular file sizes": sum_file_sizes, + } vault_cooking = { - 'directory_context': True, - 'directory_id': sha1_git, - 'revision_context': False, - 'revision_id': None + "directory_context": True, + "directory_id": sha1_git, + "revision_context": False, + "revision_id": None, } - swh_objects = [{'type': 'directory', 'id': sha1_git}] + swh_objects = [{"type": "directory", "id": sha1_git}] swh_ids = get_swh_persistent_ids( - swh_objects=swh_objects, snapshot_context=snapshot_context) + swh_objects=swh_objects, snapshot_context=snapshot_context + ) - heading = 'Directory - %s' % sha1_git + heading = "Directory - %s" % sha1_git if breadcrumbs: - dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' - heading += ' - %s' % dir_path - - return render(request, 'browse/directory.html', - {'heading': heading, - 'swh_object_id': swh_ids[0]['swh_id'], - 'swh_object_name': 'Directory', - 'swh_object_metadata': dir_metadata, - 'dirs': dirs, - 'files': files, - 'breadcrumbs': breadcrumbs, - 'top_right_link': None, - 'readme_name': readme_name, - 'readme_url': readme_url, - 'readme_html': readme_html, - 'snapshot_context': snapshot_context, - 'vault_cooking': vault_cooking, - 'show_actions_menu': True, - 'swh_ids': swh_ids}) - - -@browse_route(r'directory/resolve/content-path/(?P[0-9a-f]+)/(?P.+)/', # noqa - view_name='browse-directory-resolve-content-path', - checksum_args=['sha1_git']) + dir_path = "/".join([bc["name"] for bc in breadcrumbs]) + "/" + heading += " - %s" % dir_path + + return render( + request, + "browse/directory.html", + { + "heading": heading, + "swh_object_id": swh_ids[0]["swh_id"], + "swh_object_name": "Directory", + "swh_object_metadata": dir_metadata, + "dirs": dirs, + "files": files, + "breadcrumbs": breadcrumbs, + "top_right_link": None, + "readme_name": readme_name, + "readme_url": readme_url, + "readme_html": readme_html, + "snapshot_context": snapshot_context, + "vault_cooking": vault_cooking, + "show_actions_menu": True, + "swh_ids": swh_ids, + }, + ) + + +@browse_route( + r"directory/resolve/content-path/(?P[0-9a-f]+)/(?P.+)/", + view_name="browse-directory-resolve-content-path", + checksum_args=["sha1_git"], +) def _directory_resolve_content_path(request, sha1_git, path): """ Internal endpoint redirecting to data url for a specific file path relative to a root directory. """ try: path = os.path.normpath(path) - if not path.startswith('../'): + if not path.startswith("../"): dir_info = service.lookup_directory_with_path(sha1_git, path) - if dir_info['type'] == 'file': - sha1 = dir_info['checksums']['sha1'] - data_url = reverse('browse-content-raw', - url_args={'query_string': sha1}) + if dir_info["type"] == "file": + sha1 = dir_info["checksums"]["sha1"] + data_url = reverse( + "browse-content-raw", url_args={"query_string": sha1} + ) return redirect(data_url) except Exception as exc: sentry_sdk.capture_exception(exc) return HttpResponse(status=404) diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py index 76de4b0b..15f0007b 100644 --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -1,176 +1,189 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render, redirect from swh.web.common import service from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.utils import ( - reverse, format_utc_iso_date, parse_timestamp -) +from swh.web.common.utils import reverse, format_utc_iso_date, parse_timestamp from swh.web.common.exc import handle_view_exception from swh.web.browse.utils import get_snapshot_context from swh.web.browse.browseurls import browse_route from .utils.snapshot_context import ( - browse_snapshot_directory, browse_snapshot_content, - browse_snapshot_log, browse_snapshot_branches, - browse_snapshot_releases + browse_snapshot_directory, + browse_snapshot_content, + browse_snapshot_log, + browse_snapshot_branches, + browse_snapshot_releases, ) -@browse_route(r'origin/(?P.+)/visit/(?P.+)/directory/', - r'origin/(?P.+)/visit/(?P.+)' - '/directory/(?P.+)/', - r'origin/(?P.+)/directory/', - r'origin/(?P.+)/directory/(?P.+)/', - view_name='browse-origin-directory') -def origin_directory_browse(request, origin_url, - timestamp=None, path=None): +@browse_route( + r"origin/(?P.+)/visit/(?P.+)/directory/", + r"origin/(?P.+)/visit/(?P.+)" "/directory/(?P.+)/", + r"origin/(?P.+)/directory/", + r"origin/(?P.+)/directory/(?P.+)/", + view_name="browse-origin-directory", +) +def origin_directory_browse(request, origin_url, timestamp=None, path=None): """Django view for browsing the content of a directory associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/(origin_url)/directory/[(path)/]` * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/directory/[(path)/]` - """ # noqa - return browse_snapshot_directory(request, origin_url=origin_url, - timestamp=timestamp, path=path) + """ + return browse_snapshot_directory( + request, origin_url=origin_url, timestamp=timestamp, path=path + ) -@browse_route(r'origin/(?P.+)/visit/(?P.+)' - '/content/(?P.+)/', - r'origin/(?P.+)/content/(?P.+)/', - view_name='browse-origin-content') -def origin_content_browse(request, origin_url, path=None, - timestamp=None): +@browse_route( + r"origin/(?P.+)/visit/(?P.+)" "/content/(?P.+)/", + r"origin/(?P.+)/content/(?P.+)/", + view_name="browse-origin-content", +) +def origin_content_browse(request, origin_url, path=None, timestamp=None): """Django view that produces an HTML display of a content associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/(origin_url)/content/(path)/` * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/content/(path)/` - """ # noqa - language = request.GET.get('language', None) - return browse_snapshot_content(request, - origin_url=origin_url, timestamp=timestamp, - path=path, selected_language=language) + """ + language = request.GET.get("language", None) + return browse_snapshot_content( + request, + origin_url=origin_url, + timestamp=timestamp, + path=path, + selected_language=language, + ) PER_PAGE = 20 -@browse_route(r'origin/(?P.+)/visit/(?P.+)/log/', - r'origin/(?P.+)/log/', - view_name='browse-origin-log') +@browse_route( + r"origin/(?P.+)/visit/(?P.+)/log/", + r"origin/(?P.+)/log/", + view_name="browse-origin-log", +) def origin_log_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of revisions history (aka the commit log) associated to a software origin. The url scheme that points to it is the following: * :http:get:`/browse/origin/(origin_url)/log/` * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/log/` - """ # noqa - return browse_snapshot_log(request, - origin_url=origin_url, timestamp=timestamp) + """ + return browse_snapshot_log(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P.+)/visit/(?P.+)/branches/', - r'origin/(?P.+)/branches/', - view_name='browse-origin-branches') +@browse_route( + r"origin/(?P.+)/visit/(?P.+)/branches/", + r"origin/(?P.+)/branches/", + view_name="browse-origin-branches", +) def origin_branches_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of branches associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/(origin_url)/branches/` * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/branches/` - """ # noqa - return browse_snapshot_branches(request, - origin_url=origin_url, timestamp=timestamp) + """ + return browse_snapshot_branches(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P.+)/visit/(?P.+)/releases/', - r'origin/(?P.+)/releases/', - view_name='browse-origin-releases') +@browse_route( + r"origin/(?P.+)/visit/(?P.+)/releases/", + r"origin/(?P.+)/releases/", + view_name="browse-origin-releases", +) def origin_releases_browse(request, origin_url, timestamp=None): """Django view that produces an HTML display of the list of releases associated to an origin for a given visit. The url scheme that points to it is the following: * :http:get:`/browse/origin/(origin_url)/releases/` * :http:get:`/browse/origin/(origin_url)/visit/(timestamp)/releases/` - """ # noqa - return browse_snapshot_releases(request, - origin_url=origin_url, timestamp=timestamp) + """ + return browse_snapshot_releases(request, origin_url=origin_url, timestamp=timestamp) -@browse_route(r'origin/(?P.+)/visits/', - view_name='browse-origin-visits') +@browse_route(r"origin/(?P.+)/visits/", view_name="browse-origin-visits") def origin_visits_browse(request, origin_url): """Django view that produces an HTML display of visits reporting for a given origin. The url that points to it is :http:get:`/browse/origin/(origin_url)/visits/`. """ try: - origin_info = service.lookup_origin({'url': origin_url}) + origin_info = service.lookup_origin({"url": origin_url}) origin_visits = get_origin_visits(origin_info) snapshot_context = get_snapshot_context(origin_url=origin_url) except Exception as exc: return handle_view_exception(request, exc) for i, visit in enumerate(origin_visits): - url_date = format_utc_iso_date(visit['date'], '%Y-%m-%dT%H:%M:%SZ') - visit['fmt_date'] = format_utc_iso_date(visit['date']) + url_date = format_utc_iso_date(visit["date"], "%Y-%m-%dT%H:%M:%SZ") + visit["fmt_date"] = format_utc_iso_date(visit["date"]) query_params = {} if i < len(origin_visits) - 1: - if visit['date'] == origin_visits[i+1]['date']: - query_params = {'visit_id': visit['visit']} + if visit["date"] == origin_visits[i + 1]["date"]: + query_params = {"visit_id": visit["visit"]} if i > 0: - if visit['date'] == origin_visits[i-1]['date']: - query_params = {'visit_id': visit['visit']} + if visit["date"] == origin_visits[i - 1]["date"]: + query_params = {"visit_id": visit["visit"]} - snapshot = visit['snapshot'] if visit['snapshot'] else '' + snapshot = visit["snapshot"] if visit["snapshot"] else "" - visit['browse_url'] = reverse('browse-origin-directory', - url_args={'origin_url': origin_url, - 'timestamp': url_date}, - query_params=query_params) + visit["browse_url"] = reverse( + "browse-origin-directory", + url_args={"origin_url": origin_url, "timestamp": url_date}, + query_params=query_params, + ) if not snapshot: - visit['snapshot'] = '' - visit['date'] = parse_timestamp(visit['date']).timestamp() - - heading = 'Origin visits - %s' % origin_url - - return render(request, 'browse/origin-visits.html', - {'heading': heading, - 'swh_object_name': 'Visits', - 'swh_object_metadata': origin_info, - 'origin_visits': origin_visits, - 'origin_info': origin_info, - 'snapshot_context': snapshot_context, - 'vault_cooking': None, - 'show_actions_menu': False}) - - -@browse_route(r'origin/(?P.+)/', - view_name='browse-origin') + visit["snapshot"] = "" + visit["date"] = parse_timestamp(visit["date"]).timestamp() + + heading = "Origin visits - %s" % origin_url + + return render( + request, + "browse/origin-visits.html", + { + "heading": heading, + "swh_object_name": "Visits", + "swh_object_metadata": origin_info, + "origin_visits": origin_visits, + "origin_info": origin_info, + "snapshot_context": snapshot_context, + "vault_cooking": None, + "show_actions_menu": False, + }, + ) + + +@browse_route(r"origin/(?P.+)/", view_name="browse-origin") def origin_browse(request, origin_url): """Django view that redirects to the display of the latest archived snapshot for a given software origin. """ - last_snapshot_url = reverse('browse-origin-directory', - url_args={'origin_url': origin_url}) + last_snapshot_url = reverse( + "browse-origin-directory", url_args={"origin_url": origin_url} + ) return redirect(last_snapshot_url) diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py index f8ceb6af..01697a9c 100644 --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -1,210 +1,235 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import render import sentry_sdk from swh.web.common import service -from swh.web.common.utils import ( - reverse, format_utc_iso_date -) +from swh.web.common.utils import reverse, format_utc_iso_date from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( - gen_revision_link, get_snapshot_context, gen_link, - gen_snapshot_link, get_swh_persistent_ids, gen_directory_link, - gen_content_link, gen_release_link, gen_person_mail_link + gen_revision_link, + get_snapshot_context, + gen_link, + gen_snapshot_link, + get_swh_persistent_ids, + gen_directory_link, + gen_content_link, + gen_release_link, + gen_person_mail_link, ) -@browse_route(r'release/(?P[0-9a-f]+)/', - view_name='browse-release', - checksum_args=['sha1_git']) +@browse_route( + r"release/(?P[0-9a-f]+)/", + view_name="browse-release", + checksum_args=["sha1_git"], +) def release_browse(request, sha1_git): """ Django view that produces an HTML display of a release identified by its id. The url that points to it is :http:get:`/browse/release/(sha1_git)/`. """ try: release = service.lookup_release(sha1_git) snapshot_context = None origin_info = None - snapshot_id = request.GET.get('snapshot_id', None) - origin_url = request.GET.get('origin_url', None) + snapshot_id = request.GET.get("snapshot_id", None) + origin_url = request.GET.get("origin_url", None) if not origin_url: - origin_url = request.GET.get('origin', None) - timestamp = request.GET.get('timestamp', None) - visit_id = request.GET.get('visit_id', None) + origin_url = request.GET.get("origin", None) + timestamp = request.GET.get("timestamp", None) + visit_id = request.GET.get("visit_id", None) if origin_url: try: snapshot_context = get_snapshot_context( - snapshot_id, origin_url, timestamp, visit_id) + snapshot_id, origin_url, timestamp, visit_id + ) except NotFoundExc: - raw_rel_url = reverse('browse-release', - url_args={'sha1_git': sha1_git}) - error_message = \ - ('The Software Heritage archive has a release ' - 'with the hash you provided but the origin ' - 'mentioned in your request appears broken: %s. ' - 'Please check the URL and try again.\n\n' - 'Nevertheless, you can still browse the release ' - 'without origin information: %s' - % (gen_link(origin_url), gen_link(raw_rel_url))) + raw_rel_url = reverse("browse-release", url_args={"sha1_git": sha1_git}) + error_message = ( + "The Software Heritage archive has a release " + "with the hash you provided but the origin " + "mentioned in your request appears broken: %s. " + "Please check the URL and try again.\n\n" + "Nevertheless, you can still browse the release " + "without origin information: %s" + % (gen_link(origin_url), gen_link(raw_rel_url)) + ) raise NotFoundExc(error_message) - origin_info = snapshot_context['origin_info'] + origin_info = snapshot_context["origin_info"] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) except Exception as exc: return handle_view_exception(request, exc) release_data = {} - release_data['author'] = 'None' - if release['author']: - release_data['author'] = gen_person_mail_link(release['author']) - release_data['date'] = format_utc_iso_date(release['date']) - release_data['release'] = sha1_git - release_data['name'] = release['name'] - release_data['synthetic'] = release['synthetic'] - release_data['target'] = release['target'] - release_data['target type'] = release['target_type'] + release_data["author"] = "None" + if release["author"]: + release_data["author"] = gen_person_mail_link(release["author"]) + release_data["date"] = format_utc_iso_date(release["date"]) + release_data["release"] = sha1_git + release_data["name"] = release["name"] + release_data["synthetic"] = release["synthetic"] + release_data["target"] = release["target"] + release_data["target type"] = release["target_type"] if snapshot_context: - if release['target_type'] == 'revision': - release_data['context-independent target'] = \ - gen_revision_link(release['target']) - elif release['target_type'] == 'content': - release_data['context-independent target'] = \ - gen_content_link(release['target']) - elif release['target_type'] == 'directory': - release_data['context-independent target'] = \ - gen_directory_link(release['target']) - elif release['target_type'] == 'release': - release_data['context-independent target'] = \ - gen_release_link(release['target']) + if release["target_type"] == "revision": + release_data["context-independent target"] = gen_revision_link( + release["target"] + ) + elif release["target_type"] == "content": + release_data["context-independent target"] = gen_content_link( + release["target"] + ) + elif release["target_type"] == "directory": + release_data["context-independent target"] = gen_directory_link( + release["target"] + ) + elif release["target_type"] == "release": + release_data["context-independent target"] = gen_release_link( + release["target"] + ) release_note_lines = [] - if release['message']: - release_note_lines = release['message'].split('\n') + if release["message"]: + release_note_lines = release["message"].split("\n") vault_cooking = None rev_directory = None target_link = None - if release['target_type'] == 'revision': - target_link = gen_revision_link(release['target'], - snapshot_context=snapshot_context, - link_text=None, link_attrs=None) + if release["target_type"] == "revision": + target_link = gen_revision_link( + release["target"], + snapshot_context=snapshot_context, + link_text=None, + link_attrs=None, + ) try: - revision = service.lookup_revision(release['target']) - rev_directory = revision['directory'] + revision = service.lookup_revision(release["target"]) + rev_directory = revision["directory"] vault_cooking = { - 'directory_context': True, - 'directory_id': rev_directory, - 'revision_context': True, - 'revision_id': release['target'] + "directory_context": True, + "directory_id": rev_directory, + "revision_context": True, + "revision_id": release["target"], } except Exception as exc: sentry_sdk.capture_exception(exc) - elif release['target_type'] == 'directory': - target_link = gen_directory_link(release['target'], - snapshot_context=snapshot_context, - link_text=None, link_attrs=None) + elif release["target_type"] == "directory": + target_link = gen_directory_link( + release["target"], + snapshot_context=snapshot_context, + link_text=None, + link_attrs=None, + ) try: # check directory exists - service.lookup_directory(release['target']) + service.lookup_directory(release["target"]) vault_cooking = { - 'directory_context': True, - 'directory_id': release['target'], - 'revision_context': False, - 'revision_id': None + "directory_context": True, + "directory_id": release["target"], + "revision_context": False, + "revision_id": None, } except Exception as exc: sentry_sdk.capture_exception(exc) - elif release['target_type'] == 'content': - target_link = gen_content_link(release['target'], - snapshot_context=snapshot_context, - link_text=None, link_attrs=None) - elif release['target_type'] == 'release': - target_link = gen_release_link(release['target'], - snapshot_context=snapshot_context, - link_text=None, link_attrs=None) + elif release["target_type"] == "content": + target_link = gen_content_link( + release["target"], + snapshot_context=snapshot_context, + link_text=None, + link_attrs=None, + ) + elif release["target_type"] == "release": + target_link = gen_release_link( + release["target"], + snapshot_context=snapshot_context, + link_text=None, + link_attrs=None, + ) rev_directory_url = None if rev_directory is not None: if origin_info: rev_directory_url = reverse( - 'browse-origin-directory', - url_args={'origin_url': origin_info['url']}, - query_params={'release': release['name']}) + "browse-origin-directory", + url_args={"origin_url": origin_info["url"]}, + query_params={"release": release["name"]}, + ) elif snapshot_id: rev_directory_url = reverse( - 'browse-snapshot-directory', - url_args={'snapshot_id': snapshot_id}, - query_params={'release': release['name']}) + "browse-snapshot-directory", + url_args={"snapshot_id": snapshot_id}, + query_params={"release": release["name"]}, + ) else: rev_directory_url = reverse( - 'browse-directory', - url_args={'sha1_git': rev_directory}) + "browse-directory", url_args={"sha1_git": rev_directory} + ) directory_link = None if rev_directory_url is not None: directory_link = gen_link(rev_directory_url, rev_directory) - release['directory_link'] = directory_link - release['target_link'] = target_link + release["directory_link"] = directory_link + release["target_link"] = target_link if snapshot_context: - release_data['snapshot'] = snapshot_context['snapshot_id'] + release_data["snapshot"] = snapshot_context["snapshot_id"] if origin_info: - release_data['context-independent release'] = \ - gen_release_link(release['id']) - release_data['origin url'] = gen_link(origin_info['url'], - origin_info['url']) - browse_snapshot_link = \ - gen_snapshot_link(snapshot_context['snapshot_id']) - release_data['context-independent snapshot'] = browse_snapshot_link + release_data["context-independent release"] = gen_release_link(release["id"]) + release_data["origin url"] = gen_link(origin_info["url"], origin_info["url"]) + browse_snapshot_link = gen_snapshot_link(snapshot_context["snapshot_id"]) + release_data["context-independent snapshot"] = browse_snapshot_link - swh_objects = [{'type': 'release', - 'id': sha1_git}] + swh_objects = [{"type": "release", "id": sha1_git}] if snapshot_context: - snapshot_id = snapshot_context['snapshot_id'] + snapshot_id = snapshot_context["snapshot_id"] if snapshot_id: - swh_objects.append({'type': 'snapshot', - 'id': snapshot_id}) + swh_objects.append({"type": "snapshot", "id": snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) - note_header = 'None' + note_header = "None" if len(release_note_lines) > 0: note_header = release_note_lines[0] - release['note_header'] = note_header - release['note_body'] = '\n'.join(release_note_lines[1:]) + release["note_header"] = note_header + release["note_body"] = "\n".join(release_note_lines[1:]) - heading = 'Release - %s' % release['name'] + heading = "Release - %s" % release["name"] if snapshot_context: - context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] + context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: - context_found = 'origin: %s' % origin_info['url'] - heading += ' - %s' % context_found - - return render(request, 'browse/release.html', - {'heading': heading, - 'swh_object_id': swh_ids[0]['swh_id'], - 'swh_object_name': 'Release', - 'swh_object_metadata': release_data, - 'release': release, - 'snapshot_context': snapshot_context, - 'show_actions_menu': True, - 'breadcrumbs': None, - 'vault_cooking': vault_cooking, - 'top_right_link': None, - 'swh_ids': swh_ids}) + context_found = "origin: %s" % origin_info["url"] + heading += " - %s" % context_found + + return render( + request, + "browse/release.html", + { + "heading": heading, + "swh_object_id": swh_ids[0]["swh_id"], + "swh_object_name": "Release", + "swh_object_metadata": release_data, + "release": release, + "snapshot_context": snapshot_context, + "show_actions_menu": True, + "breadcrumbs": None, + "vault_cooking": vault_cooking, + "top_right_link": None, + "swh_ids": swh_ids, + }, + ) diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py index 61f41d7c..d72df374 100644 --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -1,525 +1,593 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import json import textwrap from django.http import HttpResponse from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape from django.utils.safestring import mark_safe from swh.model.identifiers import persistent_identifier from swh.web.common import service from swh.web.common.utils import ( - reverse, format_utc_iso_date, gen_path_info, swh_object_icons + reverse, + format_utc_iso_date, + gen_path_info, + swh_object_icons, ) from swh.web.common.exc import NotFoundExc, handle_view_exception from swh.web.browse.browseurls import browse_route from swh.web.browse.utils import ( - gen_link, gen_revision_link, gen_revision_url, - get_snapshot_context, get_revision_log_url, get_directory_entries, - gen_directory_link, request_content, prepare_content_for_display, - content_display_max_size, gen_snapshot_link, get_readme_to_display, - get_swh_persistent_ids, format_log_entries, gen_person_mail_link + gen_link, + gen_revision_link, + gen_revision_url, + get_snapshot_context, + get_revision_log_url, + get_directory_entries, + gen_directory_link, + request_content, + prepare_content_for_display, + content_display_max_size, + gen_snapshot_link, + get_readme_to_display, + get_swh_persistent_ids, + format_log_entries, + gen_person_mail_link, ) def _gen_content_url(revision, query_string, path, snapshot_context): if snapshot_context: - url_args = snapshot_context['url_args'] - url_args['path'] = path - query_params = snapshot_context['query_params'] - query_params['revision'] = revision['id'] - content_url = reverse('browse-origin-content', - url_args=url_args, - query_params=query_params) + url_args = snapshot_context["url_args"] + url_args["path"] = path + query_params = snapshot_context["query_params"] + query_params["revision"] = revision["id"] + content_url = reverse( + "browse-origin-content", url_args=url_args, query_params=query_params + ) else: - content_path = '%s/%s' % (revision['directory'], path) - content_url = reverse('browse-content', - url_args={'query_string': query_string}, - query_params={'path': content_path}) + content_path = "%s/%s" % (revision["directory"], path) + content_url = reverse( + "browse-content", + url_args={"query_string": query_string}, + query_params={"path": content_path}, + ) return content_url def _gen_diff_link(idx, diff_anchor, link_text): if idx < _max_displayed_file_diffs: return gen_link(diff_anchor, link_text) else: return link_text # TODO: put in conf _max_displayed_file_diffs = 1000 def _gen_revision_changes_list(revision, changes, snapshot_context): """ Returns a HTML string describing the file changes introduced in a revision. As this string will be displayed in the browse revision view, links to adequate file diffs are also generated. Args: revision (str): hexadecimal representation of a revision identifier changes (list): list of file changes in the revision snapshot_context (dict): optional origin context used to reverse the content urls Returns: A string to insert in a revision HTML view. """ changes_msg = [] for i, change in enumerate(changes): hasher = hashlib.sha1() - from_query_string = '' - to_query_string = '' - diff_id = 'diff-' - if change['from']: - from_query_string = 'sha1_git:' + change['from']['target'] - diff_id += change['from']['target'] + '-' + change['from_path'] - diff_id += '-' - if change['to']: - to_query_string = 'sha1_git:' + change['to']['target'] - diff_id += change['to']['target'] + change['to_path'] - change['path'] = change['to_path'] or change['from_path'] - url_args = {'from_query_string': from_query_string, - 'to_query_string': to_query_string} - query_params = {'path': change['path']} - change['diff_url'] = reverse('diff-contents', - url_args=url_args, - query_params=query_params) - - hasher.update(diff_id.encode('utf-8')) + from_query_string = "" + to_query_string = "" + diff_id = "diff-" + if change["from"]: + from_query_string = "sha1_git:" + change["from"]["target"] + diff_id += change["from"]["target"] + "-" + change["from_path"] + diff_id += "-" + if change["to"]: + to_query_string = "sha1_git:" + change["to"]["target"] + diff_id += change["to"]["target"] + change["to_path"] + change["path"] = change["to_path"] or change["from_path"] + url_args = { + "from_query_string": from_query_string, + "to_query_string": to_query_string, + } + query_params = {"path": change["path"]} + change["diff_url"] = reverse( + "diff-contents", url_args=url_args, query_params=query_params + ) + + hasher.update(diff_id.encode("utf-8")) diff_id = hasher.hexdigest() - change['id'] = diff_id - panel_diff_link = '#panel_' + diff_id - - if change['type'] == 'modify': - change['content_url'] = \ - _gen_content_url(revision, to_query_string, - change['to_path'], snapshot_context) - changes_msg.append('modified: %s' % - _gen_diff_link(i, panel_diff_link, - change['to_path'])) - elif change['type'] == 'insert': - change['content_url'] = \ - _gen_content_url(revision, to_query_string, - change['to_path'], snapshot_context) - changes_msg.append('new file: %s' % - _gen_diff_link(i, panel_diff_link, - change['to_path'])) - elif change['type'] == 'delete': - parent = service.lookup_revision(revision['parents'][0]) - change['content_url'] = \ - _gen_content_url(parent, - from_query_string, - change['from_path'], snapshot_context) - changes_msg.append('deleted: %s' % - _gen_diff_link(i, panel_diff_link, - change['from_path'])) - elif change['type'] == 'rename': - change['content_url'] = \ - _gen_content_url(revision, to_query_string, - change['to_path'], snapshot_context) - link_text = change['from_path'] + ' → ' + change['to_path'] - changes_msg.append('renamed: %s' % - _gen_diff_link(i, panel_diff_link, link_text)) + change["id"] = diff_id + panel_diff_link = "#panel_" + diff_id + + if change["type"] == "modify": + change["content_url"] = _gen_content_url( + revision, to_query_string, change["to_path"], snapshot_context + ) + changes_msg.append( + "modified: %s" % _gen_diff_link(i, panel_diff_link, change["to_path"]) + ) + elif change["type"] == "insert": + change["content_url"] = _gen_content_url( + revision, to_query_string, change["to_path"], snapshot_context + ) + changes_msg.append( + "new file: %s" % _gen_diff_link(i, panel_diff_link, change["to_path"]) + ) + elif change["type"] == "delete": + parent = service.lookup_revision(revision["parents"][0]) + change["content_url"] = _gen_content_url( + parent, from_query_string, change["from_path"], snapshot_context + ) + changes_msg.append( + "deleted: %s" + % _gen_diff_link(i, panel_diff_link, change["from_path"]) + ) + elif change["type"] == "rename": + change["content_url"] = _gen_content_url( + revision, to_query_string, change["to_path"], snapshot_context + ) + link_text = change["from_path"] + " → " + change["to_path"] + changes_msg.append( + "renamed: %s" % _gen_diff_link(i, panel_diff_link, link_text) + ) if not changes: - changes_msg.append('No changes') - return mark_safe('\n'.join(changes_msg)) + changes_msg.append("No changes") + return mark_safe("\n".join(changes_msg)) -@browse_route(r'revision/(?P[0-9a-f]+)/diff/', - view_name='diff-revision', - checksum_args=['sha1_git']) +@browse_route( + r"revision/(?P[0-9a-f]+)/diff/", + view_name="diff-revision", + checksum_args=["sha1_git"], +) def _revision_diff(request, sha1_git): """ Browse internal endpoint to compute revision diff """ try: revision = service.lookup_revision(sha1_git) snapshot_context = None - origin_url = request.GET.get('origin_url', None) + origin_url = request.GET.get("origin_url", None) if not origin_url: - origin_url = request.GET.get('origin', None) - timestamp = request.GET.get('timestamp', None) - visit_id = request.GET.get('visit_id', None) + origin_url = request.GET.get("origin", None) + timestamp = request.GET.get("timestamp", None) + visit_id = request.GET.get("visit_id", None) if origin_url: snapshot_context = get_snapshot_context( - origin_url=origin_url, timestamp=timestamp, visit_id=visit_id) + origin_url=origin_url, timestamp=timestamp, visit_id=visit_id + ) except Exception as exc: return handle_view_exception(request, exc) changes = service.diff_revision(sha1_git) - changes_msg = _gen_revision_changes_list(revision, changes, - snapshot_context) + changes_msg = _gen_revision_changes_list(revision, changes, snapshot_context) diff_data = { - 'total_nb_changes': len(changes), - 'changes': changes[:_max_displayed_file_diffs], - 'changes_msg': changes_msg + "total_nb_changes": len(changes), + "changes": changes[:_max_displayed_file_diffs], + "changes_msg": changes_msg, } - diff_data_json = json.dumps(diff_data, separators=(',', ': ')) - return HttpResponse(diff_data_json, content_type='application/json') + diff_data_json = json.dumps(diff_data, separators=(",", ": ")) + return HttpResponse(diff_data_json, content_type="application/json") NB_LOG_ENTRIES = 100 -@browse_route(r'revision/(?P[0-9a-f]+)/log/', - view_name='browse-revision-log', - checksum_args=['sha1_git']) +@browse_route( + r"revision/(?P[0-9a-f]+)/log/", + view_name="browse-revision-log", + checksum_args=["sha1_git"], +) def revision_log_browse(request, sha1_git): """ Django view that produces an HTML display of the history log for a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/log/` """ try: - per_page = int(request.GET.get('per_page', NB_LOG_ENTRIES)) - offset = int(request.GET.get('offset', 0)) - revs_ordering = request.GET.get('revs_ordering', 'committer_date') - session_key = 'rev_%s_log_ordering_%s' % (sha1_git, revs_ordering) + per_page = int(request.GET.get("per_page", NB_LOG_ENTRIES)) + offset = int(request.GET.get("offset", 0)) + revs_ordering = request.GET.get("revs_ordering", "committer_date") + session_key = "rev_%s_log_ordering_%s" % (sha1_git, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: - rev_log = rev_log_session['rev_log'] - revs_walker_state = rev_log_session['revs_walker_state'] - - if len(rev_log) < offset+per_page: - revs_walker = \ - service.get_revisions_walker(revs_ordering, sha1_git, - max_revs=offset+per_page+1, - state=revs_walker_state) - - rev_log += [rev['id'] for rev in revs_walker] + rev_log = rev_log_session["rev_log"] + revs_walker_state = rev_log_session["revs_walker_state"] + + if len(rev_log) < offset + per_page: + revs_walker = service.get_revisions_walker( + revs_ordering, + sha1_git, + max_revs=offset + per_page + 1, + state=revs_walker_state, + ) + + rev_log += [rev["id"] for rev in revs_walker] revs_walker_state = revs_walker.export_state() - revs = rev_log[offset:offset+per_page] + revs = rev_log[offset : offset + per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { - 'rev_log': rev_log, - 'revs_walker_state': revs_walker_state + "rev_log": rev_log, + "revs_walker_state": revs_walker_state, } except Exception as exc: return handle_view_exception(request, exc) - revs_ordering = request.GET.get('revs_ordering', '') + revs_ordering = request.GET.get("revs_ordering", "") prev_log_url = None if len(rev_log) > offset + per_page: - prev_log_url = reverse('browse-revision-log', - url_args={'sha1_git': sha1_git}, - query_params={'per_page': per_page, - 'offset': offset + per_page, - 'revs_ordering': revs_ordering}) + prev_log_url = reverse( + "browse-revision-log", + url_args={"sha1_git": sha1_git}, + query_params={ + "per_page": per_page, + "offset": offset + per_page, + "revs_ordering": revs_ordering, + }, + ) next_log_url = None if offset != 0: - next_log_url = reverse('browse-revision-log', - url_args={'sha1_git': sha1_git}, - query_params={'per_page': per_page, - 'offset': offset - per_page, - 'revs_ordering': revs_ordering}) + next_log_url = reverse( + "browse-revision-log", + url_args={"sha1_git": sha1_git}, + query_params={ + "per_page": per_page, + "offset": offset - per_page, + "revs_ordering": revs_ordering, + }, + ) revision_log_data = format_log_entries(revision_log, per_page) - swh_rev_id = persistent_identifier('revision', sha1_git) - - return render(request, 'browse/revision-log.html', - {'heading': 'Revision history', - 'swh_object_id': swh_rev_id, - 'swh_object_name': 'Revisions history', - 'swh_object_metadata': None, - 'revision_log': revision_log_data, - 'revs_ordering': revs_ordering, - 'next_log_url': next_log_url, - 'prev_log_url': prev_log_url, - 'breadcrumbs': None, - 'top_right_link': None, - 'snapshot_context': None, - 'vault_cooking': None, - 'show_actions_menu': True, - 'swh_ids': None}) - - -@browse_route(r'revision/(?P[0-9a-f]+)/', - r'revision/(?P[0-9a-f]+)/(?P.+)/', - view_name='browse-revision', - checksum_args=['sha1_git']) + swh_rev_id = persistent_identifier("revision", sha1_git) + + return render( + request, + "browse/revision-log.html", + { + "heading": "Revision history", + "swh_object_id": swh_rev_id, + "swh_object_name": "Revisions history", + "swh_object_metadata": None, + "revision_log": revision_log_data, + "revs_ordering": revs_ordering, + "next_log_url": next_log_url, + "prev_log_url": prev_log_url, + "breadcrumbs": None, + "top_right_link": None, + "snapshot_context": None, + "vault_cooking": None, + "show_actions_menu": True, + "swh_ids": None, + }, + ) + + +@browse_route( + r"revision/(?P[0-9a-f]+)/", + r"revision/(?P[0-9a-f]+)/(?P.+)/", + view_name="browse-revision", + checksum_args=["sha1_git"], +) def revision_browse(request, sha1_git, extra_path=None): """ Django view that produces an HTML display of a revision identified by its id. The url that points to it is :http:get:`/browse/revision/(sha1_git)/`. """ try: revision = service.lookup_revision(sha1_git) origin_info = None snapshot_context = None - origin_url = request.GET.get('origin_url', None) + origin_url = request.GET.get("origin_url", None) if not origin_url: - origin_url = request.GET.get('origin', None) - timestamp = request.GET.get('timestamp', None) - visit_id = request.GET.get('visit_id', None) - snapshot_id = request.GET.get('snapshot_id', None) - path = request.GET.get('path', None) + origin_url = request.GET.get("origin", None) + timestamp = request.GET.get("timestamp", None) + visit_id = request.GET.get("visit_id", None) + snapshot_id = request.GET.get("snapshot_id", None) + path = request.GET.get("path", None) dir_id = None dirs, files = None, None content_data = None if origin_url: try: snapshot_context = get_snapshot_context( - origin_url=origin_url, timestamp=timestamp, - visit_id=visit_id) + origin_url=origin_url, timestamp=timestamp, visit_id=visit_id + ) except NotFoundExc: - raw_rev_url = reverse('browse-revision', - url_args={'sha1_git': sha1_git}) - error_message = \ - ('The Software Heritage archive has a revision ' - 'with the hash you provided but the origin ' - 'mentioned in your request appears broken: %s. ' - 'Please check the URL and try again.\n\n' - 'Nevertheless, you can still browse the revision ' - 'without origin information: %s' - % (gen_link(origin_url), gen_link(raw_rev_url))) + raw_rev_url = reverse( + "browse-revision", url_args={"sha1_git": sha1_git} + ) + error_message = ( + "The Software Heritage archive has a revision " + "with the hash you provided but the origin " + "mentioned in your request appears broken: %s. " + "Please check the URL and try again.\n\n" + "Nevertheless, you can still browse the revision " + "without origin information: %s" + % (gen_link(origin_url), gen_link(raw_rev_url)) + ) raise NotFoundExc(error_message) - origin_info = snapshot_context['origin_info'] - snapshot_id = snapshot_context['snapshot_id'] + origin_info = snapshot_context["origin_info"] + snapshot_id = snapshot_context["snapshot_id"] elif snapshot_id: snapshot_context = get_snapshot_context(snapshot_id) if path: - file_info = \ - service.lookup_directory_with_path(revision['directory'], path) - if file_info['type'] == 'dir': - dir_id = file_info['target'] + file_info = service.lookup_directory_with_path(revision["directory"], path) + if file_info["type"] == "dir": + dir_id = file_info["target"] else: - query_string = 'sha1_git:' + file_info['target'] - content_data = request_content(query_string, - raise_if_unavailable=False) + query_string = "sha1_git:" + file_info["target"] + content_data = request_content(query_string, raise_if_unavailable=False) else: - dir_id = revision['directory'] + dir_id = revision["directory"] if dir_id: - path = '' if path is None else (path + '/') + path = "" if path is None else (path + "/") dirs, files = get_directory_entries(dir_id) except Exception as exc: return handle_view_exception(request, exc) revision_data = {} - revision_data['author'] = 'None' - if revision['author']: - author_link = gen_person_mail_link(revision['author']) - revision_data['author'] = author_link - revision_data['committer'] = 'None' - if revision['committer']: - committer_link = gen_person_mail_link(revision['committer']) - revision_data['committer'] = committer_link - revision_data['committer date'] = \ - format_utc_iso_date(revision['committer_date']) - revision_data['date'] = format_utc_iso_date(revision['date']) - revision_data['directory'] = revision['directory'] + revision_data["author"] = "None" + if revision["author"]: + author_link = gen_person_mail_link(revision["author"]) + revision_data["author"] = author_link + revision_data["committer"] = "None" + if revision["committer"]: + committer_link = gen_person_mail_link(revision["committer"]) + revision_data["committer"] = committer_link + revision_data["committer date"] = format_utc_iso_date(revision["committer_date"]) + revision_data["date"] = format_utc_iso_date(revision["date"]) + revision_data["directory"] = revision["directory"] if snapshot_context: - revision_data['snapshot'] = snapshot_id - browse_snapshot_link = \ - gen_snapshot_link(snapshot_id) - revision_data['context-independent snapshot'] = browse_snapshot_link - - revision_data['context-independent directory'] = \ - gen_directory_link(revision['directory']) - revision_data['revision'] = sha1_git - revision_data['merge'] = revision['merge'] - revision_data['metadata'] = escape(json.dumps(revision['metadata'], - sort_keys=True, - indent=4, separators=(',', ': '))) + revision_data["snapshot"] = snapshot_id + browse_snapshot_link = gen_snapshot_link(snapshot_id) + revision_data["context-independent snapshot"] = browse_snapshot_link + + revision_data["context-independent directory"] = gen_directory_link( + revision["directory"] + ) + revision_data["revision"] = sha1_git + revision_data["merge"] = revision["merge"] + revision_data["metadata"] = escape( + json.dumps( + revision["metadata"], sort_keys=True, indent=4, separators=(",", ": ") + ) + ) if origin_info: - revision_data['origin url'] = gen_link(origin_info['url'], - origin_info['url']) - revision_data['context-independent revision'] = \ - gen_revision_link(sha1_git) + revision_data["origin url"] = gen_link(origin_info["url"], origin_info["url"]) + revision_data["context-independent revision"] = gen_revision_link(sha1_git) - parents = '' - for p in revision['parents']: - parent_link = gen_revision_link(p, link_text=None, link_attrs=None, - snapshot_context=snapshot_context) - parents += parent_link + '
' + parents = "" + for p in revision["parents"]: + parent_link = gen_revision_link( + p, link_text=None, link_attrs=None, snapshot_context=snapshot_context + ) + parents += parent_link + "
" - revision_data['parents'] = mark_safe(parents) - revision_data['synthetic'] = revision['synthetic'] - revision_data['type'] = revision['type'] + revision_data["parents"] = mark_safe(parents) + revision_data["synthetic"] = revision["synthetic"] + revision_data["type"] = revision["type"] - message_lines = ['None'] - if revision['message']: - message_lines = revision['message'].split('\n') + message_lines = ["None"] + if revision["message"]: + message_lines = revision["message"].split("\n") parents = [] - for p in revision['parents']: + for p in revision["parents"]: parent_url = gen_revision_url(p, snapshot_context) - parents.append({'id': p, 'url': parent_url}) + parents.append({"id": p, "url": parent_url}) path_info = gen_path_info(path) - query_params = {'snapshot_id': snapshot_id, - 'origin': origin_url, - 'timestamp': timestamp, - 'visit_id': visit_id} + query_params = { + "snapshot_id": snapshot_id, + "origin": origin_url, + "timestamp": timestamp, + "visit_id": visit_id, + } breadcrumbs = [] - breadcrumbs.append({'name': revision['directory'][:7], - 'url': reverse('browse-revision', - url_args={'sha1_git': sha1_git}, - query_params=query_params)}) + breadcrumbs.append( + { + "name": revision["directory"][:7], + "url": reverse( + "browse-revision", + url_args={"sha1_git": sha1_git}, + query_params=query_params, + ), + } + ) for pi in path_info: - query_params['path'] = pi['path'] - breadcrumbs.append({'name': pi['name'], - 'url': reverse('browse-revision', - url_args={'sha1_git': sha1_git}, - query_params=query_params)}) + query_params["path"] = pi["path"] + breadcrumbs.append( + { + "name": pi["name"], + "url": reverse( + "browse-revision", + url_args={"sha1_git": sha1_git}, + query_params=query_params, + ), + } + ) vault_cooking = { - 'directory_context': False, - 'directory_id': None, - 'revision_context': True, - 'revision_id': sha1_git + "directory_context": False, + "directory_id": None, + "revision_context": True, + "revision_id": sha1_git, } - swh_objects = [{'type': 'revision', - 'id': sha1_git}] + swh_objects = [{"type": "revision", "id": sha1_git}] content = None content_size = None mimetype = None language = None readme_name = None readme_url = None readme_html = None readmes = {} error_code = 200 - error_message = '' - error_description = '' + error_message = "" + error_description = "" if content_data: - breadcrumbs[-1]['url'] = None - content_size = content_data['length'] - mimetype = content_data['mimetype'] - if content_data['raw_data']: + breadcrumbs[-1]["url"] = None + content_size = content_data["length"] + mimetype = content_data["mimetype"] + if content_data["raw_data"]: content_display_data = prepare_content_for_display( - content_data['raw_data'], content_data['mimetype'], path) - content = content_display_data['content_data'] - language = content_display_data['language'] - mimetype = content_display_data['mimetype'] + content_data["raw_data"], content_data["mimetype"], path + ) + content = content_display_data["content_data"] + language = content_display_data["language"] + mimetype = content_display_data["mimetype"] query_params = {} if path: - filename = path_info[-1]['name'] - query_params['filename'] = path_info[-1]['name'] - revision_data['filename'] = filename + filename = path_info[-1]["name"] + query_params["filename"] = path_info[-1]["name"] + revision_data["filename"] = filename top_right_link = { - 'url': reverse('browse-content-raw', - url_args={'query_string': query_string}, - query_params=query_params), - 'icon': swh_object_icons['content'], - 'text': 'Raw File' + "url": reverse( + "browse-content-raw", + url_args={"query_string": query_string}, + query_params=query_params, + ), + "icon": swh_object_icons["content"], + "text": "Raw File", } - swh_objects.append({'type': 'content', - 'id': file_info['target']}) + swh_objects.append({"type": "content", "id": file_info["target"]}) - error_code = content_data['error_code'] - error_message = content_data['error_message'] - error_description = content_data['error_description'] + error_code = content_data["error_code"] + error_message = content_data["error_message"] + error_description = content_data["error_description"] else: for d in dirs: - if d['type'] == 'rev': - d['url'] = reverse('browse-revision', - url_args={'sha1_git': d['target']}) + if d["type"] == "rev": + d["url"] = reverse( + "browse-revision", url_args={"sha1_git": d["target"]} + ) else: - query_params['path'] = path + d['name'] - d['url'] = reverse('browse-revision', - url_args={'sha1_git': sha1_git}, - query_params=query_params) + query_params["path"] = path + d["name"] + d["url"] = reverse( + "browse-revision", + url_args={"sha1_git": sha1_git}, + query_params=query_params, + ) for f in files: - query_params['path'] = path + f['name'] - f['url'] = reverse('browse-revision', - url_args={'sha1_git': sha1_git}, - query_params=query_params) - if f['length'] is not None: - f['length'] = filesizeformat(f['length']) - if f['name'].lower().startswith('readme'): - readmes[f['name']] = f['checksums']['sha1'] + query_params["path"] = path + f["name"] + f["url"] = reverse( + "browse-revision", + url_args={"sha1_git": sha1_git}, + query_params=query_params, + ) + if f["length"] is not None: + f["length"] = filesizeformat(f["length"]) + if f["name"].lower().startswith("readme"): + readmes[f["name"]] = f["checksums"]["sha1"] readme_name, readme_url, readme_html = get_readme_to_display(readmes) top_right_link = { - 'url': get_revision_log_url(sha1_git, snapshot_context), - 'icon': swh_object_icons['revisions history'], - 'text': 'History' + "url": get_revision_log_url(sha1_git, snapshot_context), + "icon": swh_object_icons["revisions history"], + "text": "History", } - vault_cooking['directory_context'] = True - vault_cooking['directory_id'] = dir_id + vault_cooking["directory_context"] = True + vault_cooking["directory_id"] = dir_id - swh_objects.append({'type': 'directory', - 'id': dir_id}) + swh_objects.append({"type": "directory", "id": dir_id}) - diff_revision_url = reverse('diff-revision', - url_args={'sha1_git': sha1_git}, - query_params={'origin': origin_url, - 'timestamp': timestamp, - 'visit_id': visit_id}) + diff_revision_url = reverse( + "diff-revision", + url_args={"sha1_git": sha1_git}, + query_params={ + "origin": origin_url, + "timestamp": timestamp, + "visit_id": visit_id, + }, + ) if snapshot_id: - swh_objects.append({'type': 'snapshot', - 'id': snapshot_id}) + swh_objects.append({"type": "snapshot", "id": snapshot_id}) swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) - heading = 'Revision - %s - %s' %\ - (sha1_git[:7], textwrap.shorten(message_lines[0], width=70)) + heading = "Revision - %s - %s" % ( + sha1_git[:7], + textwrap.shorten(message_lines[0], width=70), + ) if snapshot_context: - context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] + context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: - context_found = 'origin: %s' % origin_info['url'] - heading += ' - %s' % context_found - - return render(request, 'browse/revision.html', - {'heading': heading, - 'swh_object_id': swh_ids[0]['swh_id'], - 'swh_object_name': 'Revision', - 'swh_object_metadata': revision_data, - 'message_header': message_lines[0], - 'message_body': '\n'.join(message_lines[1:]), - 'parents': parents, - 'snapshot_context': snapshot_context, - 'dirs': dirs, - 'files': files, - 'content': content, - 'content_size': content_size, - 'max_content_size': content_display_max_size, - 'mimetype': mimetype, - 'language': language, - 'readme_name': readme_name, - 'readme_url': readme_url, - 'readme_html': readme_html, - 'breadcrumbs': breadcrumbs, - 'top_right_link': top_right_link, - 'vault_cooking': vault_cooking, - 'diff_revision_url': diff_revision_url, - 'show_actions_menu': True, - 'swh_ids': swh_ids, - 'error_code': error_code, - 'error_message': error_message, - 'error_description': error_description}, - status=error_code) + context_found = "origin: %s" % origin_info["url"] + heading += " - %s" % context_found + + return render( + request, + "browse/revision.html", + { + "heading": heading, + "swh_object_id": swh_ids[0]["swh_id"], + "swh_object_name": "Revision", + "swh_object_metadata": revision_data, + "message_header": message_lines[0], + "message_body": "\n".join(message_lines[1:]), + "parents": parents, + "snapshot_context": snapshot_context, + "dirs": dirs, + "files": files, + "content": content, + "content_size": content_size, + "max_content_size": content_display_max_size, + "mimetype": mimetype, + "language": language, + "readme_name": readme_name, + "readme_url": readme_url, + "readme_html": readme_html, + "breadcrumbs": breadcrumbs, + "top_right_link": top_right_link, + "vault_cooking": vault_cooking, + "diff_revision_url": diff_revision_url, + "show_actions_menu": True, + "swh_ids": swh_ids, + "error_code": error_code, + "error_message": error_message, + "error_description": error_description, + }, + status=error_code, + ) diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py index de113b8f..282a4bfc 100644 --- a/swh/web/browse/views/snapshot.py +++ b/swh/web/browse/views/snapshot.py @@ -1,102 +1,120 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import redirect from swh.web.browse.browseurls import browse_route from swh.web.common.utils import reverse from .utils.snapshot_context import ( - browse_snapshot_directory, browse_snapshot_content, - browse_snapshot_log, browse_snapshot_branches, - browse_snapshot_releases + browse_snapshot_directory, + browse_snapshot_content, + browse_snapshot_log, + browse_snapshot_branches, + browse_snapshot_releases, ) -@browse_route(r'snapshot/(?P[0-9a-f]+)/', - view_name='browse-snapshot', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/", + view_name="browse-snapshot", + checksum_args=["snapshot_id"], +) def snapshot_browse(request, snapshot_id): """Django view for browsing the content of a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/` """ - browse_snapshot_url = reverse('browse-snapshot-directory', - url_args={'snapshot_id': snapshot_id}, - query_params=request.GET) + browse_snapshot_url = reverse( + "browse-snapshot-directory", + url_args={"snapshot_id": snapshot_id}, + query_params=request.GET, + ) return redirect(browse_snapshot_url) -@browse_route(r'snapshot/(?P[0-9a-f]+)/directory/', - r'snapshot/(?P[0-9a-f]+)/directory/(?P.+)/', - view_name='browse-snapshot-directory', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/directory/", + r"snapshot/(?P[0-9a-f]+)/directory/(?P.+)/", + view_name="browse-snapshot-directory", + checksum_args=["snapshot_id"], +) def snapshot_directory_browse(request, snapshot_id, path=None): """Django view for browsing the content of a directory collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/directory/[(path)/]` """ - origin_url = request.GET.get('origin_url', None) + origin_url = request.GET.get("origin_url", None) if not origin_url: - origin_url = request.GET.get('origin', None) - return browse_snapshot_directory(request, snapshot_id=snapshot_id, - path=path, origin_url=origin_url) + origin_url = request.GET.get("origin", None) + return browse_snapshot_directory( + request, snapshot_id=snapshot_id, path=path, origin_url=origin_url + ) -@browse_route(r'snapshot/(?P[0-9a-f]+)/content/(?P.+)/', - view_name='browse-snapshot-content', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/content/(?P.+)/", + view_name="browse-snapshot-content", + checksum_args=["snapshot_id"], +) def snapshot_content_browse(request, snapshot_id, path): """Django view that produces an HTML display of a content collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/content/(path)/` """ - language = request.GET.get('language', None) - return browse_snapshot_content(request, snapshot_id=snapshot_id, path=path, - selected_language=language) + language = request.GET.get("language", None) + return browse_snapshot_content( + request, snapshot_id=snapshot_id, path=path, selected_language=language + ) -@browse_route(r'snapshot/(?P[0-9a-f]+)/log/', - view_name='browse-snapshot-log', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/log/", + view_name="browse-snapshot-log", + checksum_args=["snapshot_id"], +) def snapshot_log_browse(request, snapshot_id): """Django view that produces an HTML display of revisions history (aka the commit log) collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/log/` """ return browse_snapshot_log(request, snapshot_id=snapshot_id) -@browse_route(r'snapshot/(?P[0-9a-f]+)/branches/', - view_name='browse-snapshot-branches', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/branches/", + view_name="browse-snapshot-branches", + checksum_args=["snapshot_id"], +) def snapshot_branches_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/branches/` """ return browse_snapshot_branches(request, snapshot_id=snapshot_id) -@browse_route(r'snapshot/(?P[0-9a-f]+)/releases/', - view_name='browse-snapshot-releases', - checksum_args=['snapshot_id']) +@browse_route( + r"snapshot/(?P[0-9a-f]+)/releases/", + view_name="browse-snapshot-releases", + checksum_args=["snapshot_id"], +) def snapshot_releases_browse(request, snapshot_id): """Django view that produces an HTML display of the list of releases collected in a snapshot. The url that points to it is :http:get:`/browse/snapshot/(snapshot_id)/releases/` """ return browse_snapshot_releases(request, snapshot_id=snapshot_id) diff --git a/swh/web/browse/views/utils/snapshot_context.py b/swh/web/browse/views/utils/snapshot_context.py index 4e6c4f33..f620f7e4 100644 --- a/swh/web/browse/views/utils/snapshot_context.py +++ b/swh/web/browse/views/utils/snapshot_context.py @@ -1,936 +1,1065 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Utility module implementing Django views for browsing the archive # in a snapshot context. # Its purpose is to factorize code for the views reachable from the # /origin/.* and /snapshot/.* endpoints. from django.shortcuts import render from django.template.defaultfilters import filesizeformat from django.utils.html import escape import sentry_sdk from swh.model.identifiers import snapshot_identifier from swh.web.browse.utils import ( - get_snapshot_context, get_directory_entries, gen_directory_link, - gen_revision_link, request_content, gen_content_link, - prepare_content_for_display, content_display_max_size, - format_log_entries, gen_revision_log_link, gen_release_link, - get_readme_to_display, get_swh_persistent_ids, - gen_snapshot_link, process_snapshot_branches + get_snapshot_context, + get_directory_entries, + gen_directory_link, + gen_revision_link, + request_content, + gen_content_link, + prepare_content_for_display, + content_display_max_size, + format_log_entries, + gen_revision_log_link, + gen_release_link, + get_readme_to_display, + get_swh_persistent_ids, + gen_snapshot_link, + process_snapshot_branches, ) from swh.web.common import service, highlightjs -from swh.web.common.exc import ( - handle_view_exception, NotFoundExc -) +from swh.web.common.exc import handle_view_exception, NotFoundExc from swh.web.common.utils import ( - reverse, gen_path_info, format_utc_iso_date, swh_object_icons + reverse, + gen_path_info, + format_utc_iso_date, + swh_object_icons, ) -_empty_snapshot_id = snapshot_identifier({'branches': {}}) +_empty_snapshot_id = snapshot_identifier({"branches": {}}) def _get_branch(branches, branch_name, snapshot_id): """ Utility function to get a specific branch from a branches list. Its purpose is to get the default HEAD branch as some software origin (e.g those with svn type) does not have it. In that latter case, check if there is a master branch instead and returns it. """ - filtered_branches = [b for b in branches if b['name'] == branch_name] + filtered_branches = [b for b in branches if b["name"] == branch_name] if filtered_branches: return filtered_branches[0] - elif branch_name == 'HEAD': - filtered_branches = [b for b in branches - if b['name'].endswith('master')] + elif branch_name == "HEAD": + filtered_branches = [b for b in branches if b["name"].endswith("master")] if filtered_branches: return filtered_branches[0] elif branches: return branches[0] else: # case where a large branches list has been truncated - snp = service.lookup_snapshot(snapshot_id, - branches_from=branch_name, - branches_count=1, - target_types=['revision', 'alias']) + snp = service.lookup_snapshot( + snapshot_id, + branches_from=branch_name, + branches_count=1, + target_types=["revision", "alias"], + ) snp_branch, _ = process_snapshot_branches(snp) - if snp_branch and snp_branch[0]['name'] == branch_name: + if snp_branch and snp_branch[0]["name"] == branch_name: branches.append(snp_branch[0]) return snp_branch[0] def _get_release(releases, release_name, snapshot_id): """ Utility function to get a specific release from a releases list. Returns None if the release can not be found in the list. """ - filtered_releases = [r for r in releases if r['name'] == release_name] + filtered_releases = [r for r in releases if r["name"] == release_name] if filtered_releases: return filtered_releases[0] else: # case where a large branches list has been truncated - for branch_name in (release_name, f'refs/tags/{release_name}'): - snp = service.lookup_snapshot(snapshot_id, - branches_from=branch_name, - branches_count=1, - target_types=['release']) + for branch_name in (release_name, f"refs/tags/{release_name}"): + snp = service.lookup_snapshot( + snapshot_id, + branches_from=branch_name, + branches_count=1, + target_types=["release"], + ) _, snp_release = process_snapshot_branches(snp) - if snp_release and snp_release[0]['name'] == release_name: + if snp_release and snp_release[0]["name"] == release_name: releases.append(snp_release[0]) return snp_release[0] -def _branch_not_found(branch_type, branch, snapshot_id, snapshot_sizes, - origin_info, timestamp, visit_id): +def _branch_not_found( + branch_type, branch, snapshot_id, snapshot_sizes, origin_info, timestamp, visit_id +): """ Utility function to raise an exception when a specified branch/release can not be found. """ - if branch_type == 'branch': - branch_type = 'Branch' - branch_type_plural = 'branches' - target_type = 'revision' + if branch_type == "branch": + branch_type = "Branch" + branch_type_plural = "branches" + target_type = "revision" else: - branch_type = 'Release' - branch_type_plural = 'releases' - target_type = 'release' + branch_type = "Release" + branch_type_plural = "releases" + target_type = "release" if snapshot_id and snapshot_sizes[target_type] == 0: - msg = ('Snapshot with id %s has an empty list' - ' of %s!' % (snapshot_id, branch_type_plural)) + msg = "Snapshot with id %s has an empty list" " of %s!" % ( + snapshot_id, + branch_type_plural, + ) elif snapshot_id: - msg = ('%s %s for snapshot with id %s' - ' not found!' % (branch_type, branch, snapshot_id)) + msg = "%s %s for snapshot with id %s" " not found!" % ( + branch_type, + branch, + snapshot_id, + ) elif visit_id and snapshot_sizes[target_type] == 0: - msg = ('Origin with url %s' - ' for visit with id %s has an empty list' - ' of %s!' % (origin_info['url'], visit_id, - branch_type_plural)) + msg = ( + "Origin with url %s" + " for visit with id %s has an empty list" + " of %s!" % (origin_info["url"], visit_id, branch_type_plural) + ) elif visit_id: - msg = ('%s %s associated to visit with' - ' id %s for origin with url %s' - ' not found!' % (branch_type, branch, visit_id, - origin_info['url'])) + msg = ( + "%s %s associated to visit with" + " id %s for origin with url %s" + " not found!" % (branch_type, branch, visit_id, origin_info["url"]) + ) elif snapshot_sizes[target_type] == 0: - msg = ('Origin with url %s' - ' for visit with timestamp %s has an empty list' - ' of %s!' % (origin_info['url'], - timestamp, branch_type_plural)) + msg = ( + "Origin with url %s" + " for visit with timestamp %s has an empty list" + " of %s!" % (origin_info["url"], timestamp, branch_type_plural) + ) else: - msg = ('%s %s associated to visit with' - ' timestamp %s for origin with ' - 'url %s not found!' % (branch_type, branch, timestamp, - origin_info['url'])) + msg = ( + "%s %s associated to visit with" + " timestamp %s for origin with " + "url %s not found!" % (branch_type, branch, timestamp, origin_info["url"]) + ) raise NotFoundExc(escape(msg)) -def _process_snapshot_request(request, snapshot_id=None, - origin_url=None, timestamp=None, path=None, - browse_context='directory'): +def _process_snapshot_request( + request, + snapshot_id=None, + origin_url=None, + timestamp=None, + path=None, + browse_context="directory", +): """ Utility function to perform common input request processing for snapshot context views. """ - visit_id = request.GET.get('visit_id', None) + visit_id = request.GET.get("visit_id", None) - snapshot_context = get_snapshot_context(snapshot_id, - origin_url, timestamp, visit_id) + snapshot_context = get_snapshot_context( + snapshot_id, origin_url, timestamp, visit_id + ) - swh_type = snapshot_context['swh_type'] - origin_info = snapshot_context['origin_info'] - branches = snapshot_context['branches'] - releases = snapshot_context['releases'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] + swh_type = snapshot_context["swh_type"] + origin_info = snapshot_context["origin_info"] + branches = snapshot_context["branches"] + releases = snapshot_context["releases"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] - if snapshot_context['visit_info']: - timestamp = format_utc_iso_date(snapshot_context['visit_info']['date'], - '%Y-%m-%dT%H:%M:%SZ') - snapshot_context['timestamp'] = format_utc_iso_date( - snapshot_context['visit_info']['date']) + if snapshot_context["visit_info"]: + timestamp = format_utc_iso_date( + snapshot_context["visit_info"]["date"], "%Y-%m-%dT%H:%M:%SZ" + ) + snapshot_context["timestamp"] = format_utc_iso_date( + snapshot_context["visit_info"]["date"] + ) - browse_view_name = 'browse-' + swh_type + '-' + browse_context + browse_view_name = "browse-" + swh_type + "-" + browse_context root_sha1_git = None - revision_id = request.GET.get('revision', None) - release_name = request.GET.get('release', None) + revision_id = request.GET.get("revision", None) + release_name = request.GET.get("release", None) release_id = None branch_name = None - snapshot_sizes = snapshot_context['snapshot_sizes'] + snapshot_sizes = snapshot_context["snapshot_sizes"] snapshot_total_size = sum(snapshot_sizes.values()) if snapshot_total_size and revision_id: revision = service.lookup_revision(revision_id) - root_sha1_git = revision['directory'] - branches.append({'name': revision_id, - 'revision': revision_id, - 'directory': root_sha1_git, - 'url': None}) + root_sha1_git = revision["directory"] + branches.append( + { + "name": revision_id, + "revision": revision_id, + "directory": root_sha1_git, + "url": None, + } + ) branch_name = revision_id - query_params['revision'] = revision_id + query_params["revision"] = revision_id elif snapshot_total_size and release_name: - release = _get_release(releases, release_name, - snapshot_context['snapshot_id']) + release = _get_release(releases, release_name, snapshot_context["snapshot_id"]) try: - root_sha1_git = release['directory'] - revision_id = release['target'] - release_id = release['id'] - query_params['release'] = release_name + root_sha1_git = release["directory"] + revision_id = release["target"] + release_id = release["id"] + query_params["release"] = release_name except Exception as exc: sentry_sdk.capture_exception(exc) - _branch_not_found('release', release_name, snapshot_id, - snapshot_sizes, origin_info, timestamp, visit_id) + _branch_not_found( + "release", + release_name, + snapshot_id, + snapshot_sizes, + origin_info, + timestamp, + visit_id, + ) elif snapshot_total_size: - branch_name = request.GET.get('branch', None) + branch_name = request.GET.get("branch", None) if branch_name: - query_params['branch'] = branch_name - branch = _get_branch(branches, branch_name or 'HEAD', - snapshot_context['snapshot_id']) + query_params["branch"] = branch_name + branch = _get_branch( + branches, branch_name or "HEAD", snapshot_context["snapshot_id"] + ) try: - branch_name = branch['name'] - revision_id = branch['revision'] - root_sha1_git = branch['directory'] + branch_name = branch["name"] + revision_id = branch["revision"] + root_sha1_git = branch["directory"] except Exception as exc: sentry_sdk.capture_exception(exc) - _branch_not_found('branch', branch_name, snapshot_id, - snapshot_sizes, origin_info, timestamp, visit_id) + _branch_not_found( + "branch", + branch_name, + snapshot_id, + snapshot_sizes, + origin_info, + timestamp, + visit_id, + ) for b in branches: branch_url_args = dict(url_args) branch_query_params = dict(query_params) - if 'release' in branch_query_params: - del branch_query_params['release'] - branch_query_params['branch'] = b['name'] + if "release" in branch_query_params: + del branch_query_params["release"] + branch_query_params["branch"] = b["name"] if path: - b['path'] = path - branch_url_args['path'] = path - b['url'] = reverse(browse_view_name, - url_args=branch_url_args, - query_params=branch_query_params) + b["path"] = path + branch_url_args["path"] = path + b["url"] = reverse( + browse_view_name, url_args=branch_url_args, query_params=branch_query_params + ) for r in releases: release_url_args = dict(url_args) release_query_params = dict(query_params) - if 'branch' in release_query_params: - del release_query_params['branch'] - release_query_params['release'] = r['name'] + if "branch" in release_query_params: + del release_query_params["branch"] + release_query_params["release"] = r["name"] if path: - r['path'] = path - release_url_args['path'] = path - r['url'] = reverse(browse_view_name, - url_args=release_url_args, - query_params=release_query_params) - - snapshot_context['query_params'] = query_params - snapshot_context['root_sha1_git'] = root_sha1_git - snapshot_context['revision_id'] = revision_id - snapshot_context['branch'] = branch_name - snapshot_context['release'] = release_name - snapshot_context['release_id'] = release_id + r["path"] = path + release_url_args["path"] = path + r["url"] = reverse( + browse_view_name, + url_args=release_url_args, + query_params=release_query_params, + ) + + snapshot_context["query_params"] = query_params + snapshot_context["root_sha1_git"] = root_sha1_git + snapshot_context["revision_id"] = revision_id + snapshot_context["branch"] = branch_name + snapshot_context["release"] = release_name + snapshot_context["release_id"] = release_id return snapshot_context -def browse_snapshot_directory(request, snapshot_id=None, - origin_url=None, timestamp=None, path=None): +def browse_snapshot_directory( + request, snapshot_id=None, origin_url=None, timestamp=None, path=None +): """ Django view implementation for browsing a directory in a snapshot context. """ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_url, - timestamp, path, browse_context='directory') - - root_sha1_git = snapshot_context['root_sha1_git'] + request, + snapshot_id, + origin_url, + timestamp, + path, + browse_context="directory", + ) + + root_sha1_git = snapshot_context["root_sha1_git"] sha1_git = root_sha1_git if root_sha1_git and path: dir_info = service.lookup_directory_with_path(root_sha1_git, path) - sha1_git = dir_info['target'] + sha1_git = dir_info["target"] dirs = [] files = [] if sha1_git: dirs, files = get_directory_entries(sha1_git) except Exception as exc: return handle_view_exception(request, exc) - swh_type = snapshot_context['swh_type'] - origin_info = snapshot_context['origin_info'] - visit_info = snapshot_context['visit_info'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] - revision_id = snapshot_context['revision_id'] - snapshot_id = snapshot_context['snapshot_id'] + swh_type = snapshot_context["swh_type"] + origin_info = snapshot_context["origin_info"] + visit_info = snapshot_context["visit_info"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] + revision_id = snapshot_context["revision_id"] + snapshot_id = snapshot_context["snapshot_id"] path_info = gen_path_info(path) - browse_view_name = 'browse-' + swh_type + '-directory' + browse_view_name = "browse-" + swh_type + "-directory" breadcrumbs = [] if root_sha1_git: - breadcrumbs.append({'name': root_sha1_git[:7], - 'url': reverse(browse_view_name, - url_args=url_args, - query_params=query_params)}) + breadcrumbs.append( + { + "name": root_sha1_git[:7], + "url": reverse( + browse_view_name, url_args=url_args, query_params=query_params + ), + } + ) for pi in path_info: bc_url_args = dict(url_args) - bc_url_args['path'] = pi['path'] - breadcrumbs.append({'name': pi['name'], - 'url': reverse(browse_view_name, - url_args=bc_url_args, - query_params=query_params)}) - - path = '' if path is None else (path + '/') + bc_url_args["path"] = pi["path"] + breadcrumbs.append( + { + "name": pi["name"], + "url": reverse( + browse_view_name, url_args=bc_url_args, query_params=query_params + ), + } + ) + + path = "" if path is None else (path + "/") for d in dirs: - if d['type'] == 'rev': - d['url'] = reverse('browse-revision', - url_args={'sha1_git': d['target']}) + if d["type"] == "rev": + d["url"] = reverse("browse-revision", url_args={"sha1_git": d["target"]}) else: bc_url_args = dict(url_args) - bc_url_args['path'] = path + d['name'] - d['url'] = reverse(browse_view_name, - url_args=bc_url_args, - query_params=query_params) + bc_url_args["path"] = path + d["name"] + d["url"] = reverse( + browse_view_name, url_args=bc_url_args, query_params=query_params + ) sum_file_sizes = 0 readmes = {} - browse_view_name = 'browse-' + swh_type + '-content' + browse_view_name = "browse-" + swh_type + "-content" for f in files: bc_url_args = dict(url_args) - bc_url_args['path'] = path + f['name'] - f['url'] = reverse(browse_view_name, - url_args=bc_url_args, - query_params=query_params) - if f['length'] is not None: - sum_file_sizes += f['length'] - f['length'] = filesizeformat(f['length']) - if f['name'].lower().startswith('readme'): - readmes[f['name']] = f['checksums']['sha1'] + bc_url_args["path"] = path + f["name"] + f["url"] = reverse( + browse_view_name, url_args=bc_url_args, query_params=query_params + ) + if f["length"] is not None: + sum_file_sizes += f["length"] + f["length"] = filesizeformat(f["length"]) + if f["name"].lower().startswith("readme"): + readmes[f["name"]] = f["checksums"]["sha1"] readme_name, readme_url, readme_html = get_readme_to_display(readmes) - browse_view_name = 'browse-' + swh_type + '-log' + browse_view_name = "browse-" + swh_type + "-log" history_url = None if snapshot_id != _empty_snapshot_id: - history_url = reverse(browse_view_name, - url_args=url_args, - query_params=query_params) + history_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) nb_files = None nb_dirs = None dir_path = None if root_sha1_git: nb_files = len(files) nb_dirs = len(dirs) sum_file_sizes = filesizeformat(sum_file_sizes) - dir_path = '/' + path + dir_path = "/" + path browse_dir_link = gen_directory_link(sha1_git) browse_rev_link = gen_revision_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) - dir_metadata = {"directory": sha1_git, - "context-independent directory": browse_dir_link, - "number of regular files": nb_files, - "number of subdirectories": nb_dirs, - "sum of regular file sizes": sum_file_sizes, - "path": dir_path, - "revision": revision_id, - "context-independent revision": browse_rev_link, - "snapshot": snapshot_id, - "context-independent snapshot": browse_snp_link} + dir_metadata = { + "directory": sha1_git, + "context-independent directory": browse_dir_link, + "number of regular files": nb_files, + "number of subdirectories": nb_dirs, + "sum of regular file sizes": sum_file_sizes, + "path": dir_path, + "revision": revision_id, + "context-independent revision": browse_rev_link, + "snapshot": snapshot_id, + "context-independent snapshot": browse_snp_link, + } if origin_info: - dir_metadata['origin url'] = origin_info['url'] - dir_metadata['origin visit date'] = format_utc_iso_date( - visit_info['date']) - dir_metadata['origin visit type'] = visit_info['type'] + dir_metadata["origin url"] = origin_info["url"] + dir_metadata["origin visit date"] = format_utc_iso_date(visit_info["date"]) + dir_metadata["origin visit type"] = visit_info["type"] vault_cooking = { - 'directory_context': True, - 'directory_id': sha1_git, - 'revision_context': True, - 'revision_id': revision_id + "directory_context": True, + "directory_id": sha1_git, + "revision_context": True, + "revision_id": revision_id, } - swh_objects = [{'type': 'directory', - 'id': sha1_git}, - {'type': 'revision', - 'id': revision_id}, - {'type': 'snapshot', - 'id': snapshot_id}] + swh_objects = [ + {"type": "directory", "id": sha1_git}, + {"type": "revision", "id": revision_id}, + {"type": "snapshot", "id": snapshot_id}, + ] - release_id = snapshot_context['release_id'] + release_id = snapshot_context["release_id"] if release_id: - swh_objects.append({'type': 'release', - 'id': release_id}) + swh_objects.append({"type": "release", "id": release_id}) browse_rel_link = gen_release_link(release_id) - dir_metadata['release'] = release_id - dir_metadata['context-independent release'] = browse_rel_link + dir_metadata["release"] = release_id + dir_metadata["context-independent release"] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) - dir_path = '/'.join([bc['name'] for bc in breadcrumbs]) + '/' - context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] + dir_path = "/".join([bc["name"] for bc in breadcrumbs]) + "/" + context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: - context_found = 'origin: %s' % origin_info['url'] - heading = ('Directory - %s - %s - %s' % - (dir_path, snapshot_context['branch'], context_found)) + context_found = "origin: %s" % origin_info["url"] + heading = "Directory - %s - %s - %s" % ( + dir_path, + snapshot_context["branch"], + context_found, + ) top_right_link = None - if not snapshot_context['is_empty']: + if not snapshot_context["is_empty"]: top_right_link = { - 'url': history_url, - 'icon': swh_object_icons['revisions history'], - 'text': 'History' + "url": history_url, + "icon": swh_object_icons["revisions history"], + "text": "History", } - return render(request, 'browse/directory.html', - {'heading': heading, - 'swh_object_name': 'Directory', - 'swh_object_metadata': dir_metadata, - 'dirs': dirs, - 'files': files, - 'breadcrumbs': breadcrumbs if root_sha1_git else [], - 'top_right_link': top_right_link, - 'readme_name': readme_name, - 'readme_url': readme_url, - 'readme_html': readme_html, - 'snapshot_context': snapshot_context, - 'vault_cooking': vault_cooking, - 'show_actions_menu': True, - 'swh_ids': swh_ids}) - - -def browse_snapshot_content(request, snapshot_id=None, - origin_url=None, timestamp=None, path=None, - selected_language=None): + return render( + request, + "browse/directory.html", + { + "heading": heading, + "swh_object_name": "Directory", + "swh_object_metadata": dir_metadata, + "dirs": dirs, + "files": files, + "breadcrumbs": breadcrumbs if root_sha1_git else [], + "top_right_link": top_right_link, + "readme_name": readme_name, + "readme_url": readme_url, + "readme_html": readme_html, + "snapshot_context": snapshot_context, + "vault_cooking": vault_cooking, + "show_actions_menu": True, + "swh_ids": swh_ids, + }, + ) + + +def browse_snapshot_content( + request, + snapshot_id=None, + origin_url=None, + timestamp=None, + path=None, + selected_language=None, +): """ Django view implementation for browsing a content in a snapshot context. """ try: - snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_url, - timestamp, path, - browse_context='content') + snapshot_context = _process_snapshot_request( + request, snapshot_id, origin_url, timestamp, path, browse_context="content" + ) - root_sha1_git = snapshot_context['root_sha1_git'] + root_sha1_git = snapshot_context["root_sha1_git"] sha1_git = None query_string = None content_data = None directory_id = None - split_path = path.split('/') + split_path = path.split("/") filename = split_path[-1] - filepath = path[:-len(filename)] + filepath = path[: -len(filename)] if root_sha1_git: - content_info = service.lookup_directory_with_path(root_sha1_git, - path) - sha1_git = content_info['target'] - query_string = 'sha1_git:' + sha1_git - content_data = request_content(query_string, - raise_if_unavailable=False) + content_info = service.lookup_directory_with_path(root_sha1_git, path) + sha1_git = content_info["target"] + query_string = "sha1_git:" + sha1_git + content_data = request_content(query_string, raise_if_unavailable=False) if filepath: - dir_info = service.lookup_directory_with_path(root_sha1_git, - filepath) - directory_id = dir_info['target'] + dir_info = service.lookup_directory_with_path(root_sha1_git, filepath) + directory_id = dir_info["target"] else: directory_id = root_sha1_git except Exception as exc: return handle_view_exception(request, exc) - swh_type = snapshot_context['swh_type'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] - revision_id = snapshot_context['revision_id'] - origin_info = snapshot_context['origin_info'] - visit_info = snapshot_context['visit_info'] - snapshot_id = snapshot_context['snapshot_id'] + swh_type = snapshot_context["swh_type"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] + revision_id = snapshot_context["revision_id"] + origin_info = snapshot_context["origin_info"] + visit_info = snapshot_context["visit_info"] + snapshot_id = snapshot_context["snapshot_id"] content = None language = None mimetype = None - if content_data and content_data['raw_data'] is not None: + if content_data and content_data["raw_data"] is not None: content_display_data = prepare_content_for_display( - content_data['raw_data'], content_data['mimetype'], path) - content = content_display_data['content_data'] - language = content_display_data['language'] - mimetype = content_display_data['mimetype'] + content_data["raw_data"], content_data["mimetype"], path + ) + content = content_display_data["content_data"] + language = content_display_data["language"] + mimetype = content_display_data["mimetype"] # Override language with user-selected language if selected_language is not None: language = selected_language available_languages = None - if mimetype and 'text/' in mimetype: + if mimetype and "text/" in mimetype: available_languages = highlightjs.get_supported_languages() - browse_view_name = 'browse-' + swh_type + '-directory' + browse_view_name = "browse-" + swh_type + "-directory" breadcrumbs = [] path_info = gen_path_info(filepath) if root_sha1_git: - breadcrumbs.append({'name': root_sha1_git[:7], - 'url': reverse(browse_view_name, - url_args=url_args, - query_params=query_params)}) + breadcrumbs.append( + { + "name": root_sha1_git[:7], + "url": reverse( + browse_view_name, url_args=url_args, query_params=query_params + ), + } + ) for pi in path_info: bc_url_args = dict(url_args) - bc_url_args['path'] = pi['path'] - breadcrumbs.append({'name': pi['name'], - 'url': reverse(browse_view_name, - url_args=bc_url_args, - query_params=query_params)}) - - breadcrumbs.append({'name': filename, - 'url': None}) + bc_url_args["path"] = pi["path"] + breadcrumbs.append( + { + "name": pi["name"], + "url": reverse( + browse_view_name, url_args=bc_url_args, query_params=query_params + ), + } + ) + + breadcrumbs.append({"name": filename, "url": None}) browse_content_link = gen_content_link(sha1_git) content_raw_url = None if query_string: - content_raw_url = reverse('browse-content-raw', - url_args={'query_string': query_string}, - query_params={'filename': filename}) + content_raw_url = reverse( + "browse-content-raw", + url_args={"query_string": query_string}, + query_params={"filename": filename}, + ) browse_rev_link = gen_revision_link(revision_id) browse_dir_link = gen_directory_link(directory_id) content_metadata = { - 'context-independent content': browse_content_link, - 'path': None, - 'filename': None, - 'directory': directory_id, - 'context-independent directory': browse_dir_link, - 'revision': revision_id, - 'context-independent revision': browse_rev_link, - 'snapshot': snapshot_id + "context-independent content": browse_content_link, + "path": None, + "filename": None, + "directory": directory_id, + "context-independent directory": browse_dir_link, + "revision": revision_id, + "context-independent revision": browse_rev_link, + "snapshot": snapshot_id, } cnt_sha1_git = None content_size = None error_code = 200 - error_description = '' - error_message = '' + error_description = "" + error_message = "" if content_data: - for checksum in content_data['checksums'].keys(): - content_metadata[checksum] = content_data['checksums'][checksum] - content_metadata['mimetype'] = content_data['mimetype'] - content_metadata['encoding'] = content_data['encoding'] - content_metadata['size'] = filesizeformat(content_data['length']) - content_metadata['language'] = content_data['language'] - content_metadata['licenses'] = content_data['licenses'] - content_metadata['path'] = '/' + filepath - content_metadata['filename'] = filename - - cnt_sha1_git = content_data['checksums']['sha1_git'] - content_size = content_data['length'] - error_code = content_data['error_code'] - error_message = content_data['error_message'] - error_description = content_data['error_description'] + for checksum in content_data["checksums"].keys(): + content_metadata[checksum] = content_data["checksums"][checksum] + content_metadata["mimetype"] = content_data["mimetype"] + content_metadata["encoding"] = content_data["encoding"] + content_metadata["size"] = filesizeformat(content_data["length"]) + content_metadata["language"] = content_data["language"] + content_metadata["licenses"] = content_data["licenses"] + content_metadata["path"] = "/" + filepath + content_metadata["filename"] = filename + + cnt_sha1_git = content_data["checksums"]["sha1_git"] + content_size = content_data["length"] + error_code = content_data["error_code"] + error_message = content_data["error_message"] + error_description = content_data["error_description"] if origin_info: - content_metadata['origin url'] = origin_info['url'] - content_metadata['origin visit date'] = format_utc_iso_date( - visit_info['date']) - content_metadata['origin visit type'] = visit_info['type'] + content_metadata["origin url"] = origin_info["url"] + content_metadata["origin visit date"] = format_utc_iso_date(visit_info["date"]) + content_metadata["origin visit type"] = visit_info["type"] browse_snapshot_link = gen_snapshot_link(snapshot_id) - content_metadata['context-independent snapshot'] = browse_snapshot_link - - swh_objects = [{'type': 'content', - 'id': cnt_sha1_git}, - {'type': 'directory', - 'id': directory_id}, - {'type': 'revision', - 'id': revision_id}, - {'type': 'snapshot', - 'id': snapshot_id}] - - release_id = snapshot_context['release_id'] + content_metadata["context-independent snapshot"] = browse_snapshot_link + + swh_objects = [ + {"type": "content", "id": cnt_sha1_git}, + {"type": "directory", "id": directory_id}, + {"type": "revision", "id": revision_id}, + {"type": "snapshot", "id": snapshot_id}, + ] + + release_id = snapshot_context["release_id"] if release_id: - swh_objects.append({'type': 'release', - 'id': release_id}) + swh_objects.append({"type": "release", "id": release_id}) browse_rel_link = gen_release_link(release_id) - content_metadata['release'] = release_id - content_metadata['context-independent release'] = browse_rel_link + content_metadata["release"] = release_id + content_metadata["context-independent release"] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) - content_path = '/'.join([bc['name'] for bc in breadcrumbs]) - context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] + content_path = "/".join([bc["name"] for bc in breadcrumbs]) + context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: - context_found = 'origin: %s' % origin_info['url'] - heading = ('Content - %s - %s - %s' % - (content_path, snapshot_context['branch'], context_found)) + context_found = "origin: %s" % origin_info["url"] + heading = "Content - %s - %s - %s" % ( + content_path, + snapshot_context["branch"], + context_found, + ) top_right_link = None - if not snapshot_context['is_empty']: + if not snapshot_context["is_empty"]: top_right_link = { - 'url': content_raw_url, - 'icon': swh_object_icons['content'], - 'text': 'Raw File' + "url": content_raw_url, + "icon": swh_object_icons["content"], + "text": "Raw File", } - return render(request, 'browse/content.html', - {'heading': heading, - 'swh_object_name': 'Content', - 'swh_object_metadata': content_metadata, - 'content': content, - 'content_size': content_size, - 'max_content_size': content_display_max_size, - 'mimetype': mimetype, - 'language': language, - 'available_languages': available_languages, - 'breadcrumbs': breadcrumbs if root_sha1_git else [], - 'top_right_link': top_right_link, - 'snapshot_context': snapshot_context, - 'vault_cooking': None, - 'show_actions_menu': True, - 'swh_ids': swh_ids, - 'error_code': error_code, - 'error_message': error_message, - 'error_description': error_description}, - status=error_code) + return render( + request, + "browse/content.html", + { + "heading": heading, + "swh_object_name": "Content", + "swh_object_metadata": content_metadata, + "content": content, + "content_size": content_size, + "max_content_size": content_display_max_size, + "mimetype": mimetype, + "language": language, + "available_languages": available_languages, + "breadcrumbs": breadcrumbs if root_sha1_git else [], + "top_right_link": top_right_link, + "snapshot_context": snapshot_context, + "vault_cooking": None, + "show_actions_menu": True, + "swh_ids": swh_ids, + "error_code": error_code, + "error_message": error_message, + "error_description": error_description, + }, + status=error_code, + ) PER_PAGE = 100 -def browse_snapshot_log(request, snapshot_id=None, - origin_url=None, timestamp=None): +def browse_snapshot_log(request, snapshot_id=None, origin_url=None, timestamp=None): """ Django view implementation for browsing a revision history in a snapshot context. """ try: snapshot_context = _process_snapshot_request( - request, snapshot_id, origin_url, timestamp, browse_context='log') + request, snapshot_id, origin_url, timestamp, browse_context="log" + ) - revision_id = snapshot_context['revision_id'] + revision_id = snapshot_context["revision_id"] - per_page = int(request.GET.get('per_page', PER_PAGE)) - offset = int(request.GET.get('offset', 0)) - revs_ordering = request.GET.get('revs_ordering', 'committer_date') - session_key = 'rev_%s_log_ordering_%s' % (revision_id, revs_ordering) + per_page = int(request.GET.get("per_page", PER_PAGE)) + offset = int(request.GET.get("offset", 0)) + revs_ordering = request.GET.get("revs_ordering", "committer_date") + session_key = "rev_%s_log_ordering_%s" % (revision_id, revs_ordering) rev_log_session = request.session.get(session_key, None) rev_log = [] revs_walker_state = None if rev_log_session: - rev_log = rev_log_session['rev_log'] - revs_walker_state = rev_log_session['revs_walker_state'] + rev_log = rev_log_session["rev_log"] + revs_walker_state = rev_log_session["revs_walker_state"] - if len(rev_log) < offset+per_page: + if len(rev_log) < offset + per_page: revs_walker = service.get_revisions_walker( - revs_ordering, revision_id, - max_revs=offset+per_page+1, - state=revs_walker_state) - rev_log += [rev['id'] for rev in revs_walker] + revs_ordering, + revision_id, + max_revs=offset + per_page + 1, + state=revs_walker_state, + ) + rev_log += [rev["id"] for rev in revs_walker] revs_walker_state = revs_walker.export_state() - revs = rev_log[offset:offset+per_page] + revs = rev_log[offset : offset + per_page] revision_log = service.lookup_revision_multiple(revs) request.session[session_key] = { - 'rev_log': rev_log, - 'revs_walker_state': revs_walker_state + "rev_log": rev_log, + "revs_walker_state": revs_walker_state, } except Exception as exc: return handle_view_exception(request, exc) - swh_type = snapshot_context['swh_type'] - origin_info = snapshot_context['origin_info'] - visit_info = snapshot_context['visit_info'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] - snapshot_id = snapshot_context['snapshot_id'] + swh_type = snapshot_context["swh_type"] + origin_info = snapshot_context["origin_info"] + visit_info = snapshot_context["visit_info"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] + snapshot_id = snapshot_context["snapshot_id"] - query_params['per_page'] = per_page - revs_ordering = request.GET.get('revs_ordering', '') - query_params['revs_ordering'] = revs_ordering + query_params["per_page"] = per_page + revs_ordering = request.GET.get("revs_ordering", "") + query_params["revs_ordering"] = revs_ordering - browse_view_name = 'browse-' + swh_type + '-log' + browse_view_name = "browse-" + swh_type + "-log" prev_log_url = None if len(rev_log) > offset + per_page: - query_params['offset'] = offset + per_page - prev_log_url = reverse(browse_view_name, - url_args=url_args, - query_params=query_params) + query_params["offset"] = offset + per_page + prev_log_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) next_log_url = None if offset != 0: - query_params['offset'] = offset - per_page - next_log_url = reverse(browse_view_name, - url_args=url_args, - query_params=query_params) + query_params["offset"] = offset - per_page + next_log_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) - revision_log_data = format_log_entries(revision_log, per_page, - snapshot_context) + revision_log_data = format_log_entries(revision_log, per_page, snapshot_context) browse_rev_link = gen_revision_link(revision_id) browse_log_link = gen_revision_log_link(revision_id) browse_snp_link = gen_snapshot_link(snapshot_id) revision_metadata = { - 'context-independent revision': browse_rev_link, - 'context-independent revision history': browse_log_link, - 'context-independent snapshot': browse_snp_link, - 'snapshot': snapshot_id + "context-independent revision": browse_rev_link, + "context-independent revision history": browse_log_link, + "context-independent snapshot": browse_snp_link, + "snapshot": snapshot_id, } if origin_info: - revision_metadata['origin url'] = origin_info['url'] - revision_metadata['origin visit date'] = format_utc_iso_date( - visit_info['date']) - revision_metadata['origin visit type'] = visit_info['type'] + revision_metadata["origin url"] = origin_info["url"] + revision_metadata["origin visit date"] = format_utc_iso_date(visit_info["date"]) + revision_metadata["origin visit type"] = visit_info["type"] - swh_objects = [{'type': 'revision', - 'id': revision_id}, - {'type': 'snapshot', - 'id': snapshot_id}] + swh_objects = [ + {"type": "revision", "id": revision_id}, + {"type": "snapshot", "id": snapshot_id}, + ] - release_id = snapshot_context['release_id'] + release_id = snapshot_context["release_id"] if release_id: - swh_objects.append({'type': 'release', - 'id': release_id}) + swh_objects.append({"type": "release", "id": release_id}) browse_rel_link = gen_release_link(release_id) - revision_metadata['release'] = release_id - revision_metadata['context-independent release'] = browse_rel_link + revision_metadata["release"] = release_id + revision_metadata["context-independent release"] = browse_rel_link swh_ids = get_swh_persistent_ids(swh_objects, snapshot_context) - context_found = 'snapshot: %s' % snapshot_context['snapshot_id'] + context_found = "snapshot: %s" % snapshot_context["snapshot_id"] if origin_info: - context_found = 'origin: %s' % origin_info['url'] - heading = ('Revision history - %s - %s' % - (snapshot_context['branch'], context_found)) - - return render(request, 'browse/revision-log.html', - {'heading': heading, - 'swh_object_name': 'Revisions history', - 'swh_object_metadata': revision_metadata, - 'revision_log': revision_log_data, - 'revs_ordering': revs_ordering, - 'next_log_url': next_log_url, - 'prev_log_url': prev_log_url, - 'breadcrumbs': None, - 'top_right_link': None, - 'snapshot_context': snapshot_context, - 'vault_cooking': None, - 'show_actions_menu': True, - 'swh_ids': swh_ids}) - - -def browse_snapshot_branches(request, snapshot_id=None, - origin_url=None, timestamp=None): + context_found = "origin: %s" % origin_info["url"] + heading = "Revision history - %s - %s" % (snapshot_context["branch"], context_found) + + return render( + request, + "browse/revision-log.html", + { + "heading": heading, + "swh_object_name": "Revisions history", + "swh_object_metadata": revision_metadata, + "revision_log": revision_log_data, + "revs_ordering": revs_ordering, + "next_log_url": next_log_url, + "prev_log_url": prev_log_url, + "breadcrumbs": None, + "top_right_link": None, + "snapshot_context": snapshot_context, + "vault_cooking": None, + "show_actions_menu": True, + "swh_ids": swh_ids, + }, + ) + + +def browse_snapshot_branches( + request, snapshot_id=None, origin_url=None, timestamp=None +): """ Django view implementation for browsing a list of branches in a snapshot context. """ try: - snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_url, timestamp) + snapshot_context = _process_snapshot_request( + request, snapshot_id, origin_url, timestamp + ) - branches_bc = request.GET.get('branches_breadcrumbs', '') - branches_bc = branches_bc.split(',') if branches_bc else [] - branches_from = branches_bc[-1] if branches_bc else '' + branches_bc = request.GET.get("branches_breadcrumbs", "") + branches_bc = branches_bc.split(",") if branches_bc else [] + branches_from = branches_bc[-1] if branches_bc else "" - swh_type = snapshot_context['swh_type'] - origin_info = snapshot_context['origin_info'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] + swh_type = snapshot_context["swh_type"] + origin_info = snapshot_context["origin_info"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] - browse_view_name = 'browse-' + swh_type + '-directory' + browse_view_name = "browse-" + swh_type + "-directory" - snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], - branches_from, PER_PAGE+1, - target_types=['revision', 'alias']) + snapshot = service.lookup_snapshot( + snapshot_context["snapshot_id"], + branches_from, + PER_PAGE + 1, + target_types=["revision", "alias"], + ) displayed_branches, _ = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for branch in displayed_branches: if snapshot_id: - revision_url = reverse('browse-revision', - url_args={'sha1_git': branch['revision']}, - query_params={'snapshot_id': snapshot_id}) + revision_url = reverse( + "browse-revision", + url_args={"sha1_git": branch["revision"]}, + query_params={"snapshot_id": snapshot_id}, + ) else: - revision_url = reverse('browse-revision', - url_args={'sha1_git': branch['revision']}, - query_params={'origin': origin_info['url']}) - query_params['branch'] = branch['name'] - directory_url = reverse(browse_view_name, - url_args=url_args, - query_params=query_params) - del query_params['branch'] - branch['revision_url'] = revision_url - branch['directory_url'] = directory_url - - browse_view_name = 'browse-' + swh_type + '-branches' + revision_url = reverse( + "browse-revision", + url_args={"sha1_git": branch["revision"]}, + query_params={"origin": origin_info["url"]}, + ) + query_params["branch"] = branch["name"] + directory_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) + del query_params["branch"] + branch["revision_url"] = revision_url + branch["directory_url"] = directory_url + + browse_view_name = "browse-" + swh_type + "-branches" prev_branches_url = None next_branches_url = None if branches_bc: query_params_prev = dict(query_params) - query_params_prev['branches_breadcrumbs'] = ','.join(branches_bc[:-1]) - prev_branches_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params_prev) + query_params_prev["branches_breadcrumbs"] = ",".join(branches_bc[:-1]) + prev_branches_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params_prev + ) elif branches_from: - prev_branches_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params) + prev_branches_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) - if snapshot['next_branch'] is not None: + if snapshot["next_branch"] is not None: query_params_next = dict(query_params) - next_branch = displayed_branches[-1]['name'] + next_branch = displayed_branches[-1]["name"] del displayed_branches[-1] branches_bc.append(next_branch) - query_params_next['branches_breadcrumbs'] = ','.join(branches_bc) - next_branches_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params_next) + query_params_next["branches_breadcrumbs"] = ",".join(branches_bc) + next_branches_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params_next + ) - heading = 'Branches - ' + heading = "Branches - " if origin_info: - heading += 'origin: %s' % origin_info['url'] + heading += "origin: %s" % origin_info["url"] else: - heading += 'snapshot: %s' % snapshot_id - - return render(request, 'browse/branches.html', - {'heading': heading, - 'swh_object_name': 'Branches', - 'swh_object_metadata': {}, - 'top_right_link': None, - 'displayed_branches': displayed_branches, - 'prev_branches_url': prev_branches_url, - 'next_branches_url': next_branches_url, - 'snapshot_context': snapshot_context}) - - -def browse_snapshot_releases(request, snapshot_id=None, - origin_url=None, timestamp=None): + heading += "snapshot: %s" % snapshot_id + + return render( + request, + "browse/branches.html", + { + "heading": heading, + "swh_object_name": "Branches", + "swh_object_metadata": {}, + "top_right_link": None, + "displayed_branches": displayed_branches, + "prev_branches_url": prev_branches_url, + "next_branches_url": next_branches_url, + "snapshot_context": snapshot_context, + }, + ) + + +def browse_snapshot_releases( + request, snapshot_id=None, origin_url=None, timestamp=None +): """ Django view implementation for browsing a list of releases in a snapshot context. """ try: - snapshot_context = _process_snapshot_request(request, snapshot_id, - origin_url, timestamp) + snapshot_context = _process_snapshot_request( + request, snapshot_id, origin_url, timestamp + ) - rel_bc = request.GET.get('releases_breadcrumbs', '') - rel_bc = rel_bc.split(',') if rel_bc else [] - rel_from = rel_bc[-1] if rel_bc else '' + rel_bc = request.GET.get("releases_breadcrumbs", "") + rel_bc = rel_bc.split(",") if rel_bc else [] + rel_from = rel_bc[-1] if rel_bc else "" - swh_type = snapshot_context['swh_type'] - origin_info = snapshot_context['origin_info'] - url_args = snapshot_context['url_args'] - query_params = snapshot_context['query_params'] + swh_type = snapshot_context["swh_type"] + origin_info = snapshot_context["origin_info"] + url_args = snapshot_context["url_args"] + query_params = snapshot_context["query_params"] - snapshot = service.lookup_snapshot(snapshot_context['snapshot_id'], - rel_from, PER_PAGE+1, - target_types=['release', 'alias']) + snapshot = service.lookup_snapshot( + snapshot_context["snapshot_id"], + rel_from, + PER_PAGE + 1, + target_types=["release", "alias"], + ) _, displayed_releases = process_snapshot_branches(snapshot) except Exception as exc: return handle_view_exception(request, exc) for release in displayed_releases: if snapshot_id: - query_params_tgt = {'snapshot_id': snapshot_id} + query_params_tgt = {"snapshot_id": snapshot_id} else: - query_params_tgt = {'origin': origin_info['url']} - release_url = reverse('browse-release', - url_args={'sha1_git': release['id']}, - query_params=query_params_tgt) - - target_url = '' - if release['target_type'] == 'revision': - target_url = reverse('browse-revision', - url_args={'sha1_git': release['target']}, - query_params=query_params_tgt) - elif release['target_type'] == 'directory': - target_url = reverse('browse-directory', - url_args={'sha1_git': release['target']}, - query_params=query_params_tgt) - elif release['target_type'] == 'content': - target_url = reverse('browse-content', - url_args={'query_string': release['target']}, - query_params=query_params_tgt) - elif release['target_type'] == 'release': - target_url = reverse('browse-release', - url_args={'sha1_git': release['target']}, - query_params=query_params_tgt) - - release['release_url'] = release_url - release['target_url'] = target_url - - browse_view_name = 'browse-' + swh_type + '-releases' + query_params_tgt = {"origin": origin_info["url"]} + release_url = reverse( + "browse-release", + url_args={"sha1_git": release["id"]}, + query_params=query_params_tgt, + ) + + target_url = "" + if release["target_type"] == "revision": + target_url = reverse( + "browse-revision", + url_args={"sha1_git": release["target"]}, + query_params=query_params_tgt, + ) + elif release["target_type"] == "directory": + target_url = reverse( + "browse-directory", + url_args={"sha1_git": release["target"]}, + query_params=query_params_tgt, + ) + elif release["target_type"] == "content": + target_url = reverse( + "browse-content", + url_args={"query_string": release["target"]}, + query_params=query_params_tgt, + ) + elif release["target_type"] == "release": + target_url = reverse( + "browse-release", + url_args={"sha1_git": release["target"]}, + query_params=query_params_tgt, + ) + + release["release_url"] = release_url + release["target_url"] = target_url + + browse_view_name = "browse-" + swh_type + "-releases" prev_releases_url = None next_releases_url = None if rel_bc: query_params_prev = dict(query_params) - query_params_prev['releases_breadcrumbs'] = ','.join(rel_bc[:-1]) - prev_releases_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params_prev) + query_params_prev["releases_breadcrumbs"] = ",".join(rel_bc[:-1]) + prev_releases_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params_prev + ) elif rel_from: - prev_releases_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params) + prev_releases_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params + ) - if snapshot['next_branch'] is not None: + if snapshot["next_branch"] is not None: query_params_next = dict(query_params) - next_rel = displayed_releases[-1]['branch_name'] + next_rel = displayed_releases[-1]["branch_name"] del displayed_releases[-1] rel_bc.append(next_rel) - query_params_next['releases_breadcrumbs'] = ','.join(rel_bc) - next_releases_url = reverse(browse_view_name, url_args=url_args, - query_params=query_params_next) + query_params_next["releases_breadcrumbs"] = ",".join(rel_bc) + next_releases_url = reverse( + browse_view_name, url_args=url_args, query_params=query_params_next + ) - heading = 'Releases - ' + heading = "Releases - " if origin_info: - heading += 'origin: %s' % origin_info['url'] + heading += "origin: %s" % origin_info["url"] else: - heading += 'snapshot: %s' % snapshot_id - - return render(request, 'browse/releases.html', - {'heading': heading, - 'top_panel_visible': False, - 'top_panel_collapsible': False, - 'swh_object_name': 'Releases', - 'swh_object_metadata': {}, - 'top_right_link': None, - 'displayed_releases': displayed_releases, - 'prev_releases_url': prev_releases_url, - 'next_releases_url': next_releases_url, - 'snapshot_context': snapshot_context, - 'vault_cooking': None, - 'show_actions_menu': False}) + heading += "snapshot: %s" % snapshot_id + + return render( + request, + "browse/releases.html", + { + "heading": heading, + "top_panel_visible": False, + "top_panel_collapsible": False, + "swh_object_name": "Releases", + "swh_object_metadata": {}, + "top_right_link": None, + "displayed_releases": displayed_releases, + "prev_releases_url": prev_releases_url, + "next_releases_url": next_releases_url, + "snapshot_context": snapshot_context, + "vault_cooking": None, + "show_actions_menu": False, + }, + ) diff --git a/swh/web/common/__init__.py b/swh/web/common/__init__.py index da757d63..80eb6395 100644 --- a/swh/web/common/__init__.py +++ b/swh/web/common/__init__.py @@ -1,6 +1,6 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -default_app_config = 'swh.web.common.apps.SwhWebCommonConfig' +default_app_config = "swh.web.common.apps.SwhWebCommonConfig" diff --git a/swh/web/common/apps.py b/swh/web/common/apps.py index 52c13450..f5727de8 100644 --- a/swh/web/common/apps.py +++ b/swh/web/common/apps.py @@ -1,11 +1,11 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.apps import AppConfig class SwhWebCommonConfig(AppConfig): - name = 'swh.web.common' - label = 'swh.web.common' + name = "swh.web.common" + label = "swh.web.common" diff --git a/swh/web/common/converters.py b/swh/web/common/converters.py index 71072176..e510e9ac 100644 --- a/swh/web/common/converters.py +++ b/swh/web/common/converters.py @@ -1,375 +1,387 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json from swh.model import hashutil from swh.core.utils import decode_with_escape def _group_checksums(data): """Groups checksums values computed from hash functions used in swh and stored in data dict under a single entry 'checksums' """ if data: checksums = {} for hash in hashutil.ALGORITHMS: if hash in data and data[hash]: checksums[hash] = data[hash] del data[hash] if len(checksums) > 0: - data['checksums'] = checksums + data["checksums"] = checksums def fmap(f, data): """Map f to data at each level. This must keep the origin data structure type: - map -> map - dict -> dict - list -> list - None -> None Args: f: function that expects one argument. data: data to traverse to apply the f function. list, map, dict or bare value. Returns: The same data-structure with modified values by the f function. """ if data is None: return data if isinstance(data, map): return map(lambda y: fmap(f, y), (x for x in data)) if isinstance(data, list): return [fmap(f, x) for x in data] if isinstance(data, dict): return {k: fmap(f, v) for (k, v) in data.items()} return f(data) -def from_swh(dict_swh, hashess={}, bytess={}, dates={}, blacklist={}, - removables_if_empty={}, empty_dict={}, empty_list={}, - convert={}, convert_fn=lambda x: x): +def from_swh( + dict_swh, + hashess={}, + bytess={}, + dates={}, + blacklist={}, + removables_if_empty={}, + empty_dict={}, + empty_list={}, + convert={}, + convert_fn=lambda x: x, +): """Convert from a swh dictionary to something reasonably json serializable. Args: dict_swh: the origin dictionary needed to be transformed hashess: list/set of keys representing hashes values (sha1, sha256, sha1_git, etc...) as bytes. Those need to be transformed in hexadecimal string bytess: list/set of keys representing bytes values which needs to be decoded blacklist: set of keys to filter out from the conversion convert: set of keys whose associated values need to be converted using convert_fn convert_fn: the conversion function to apply on the value of key in 'convert' The remaining keys are copied as is in the output. Returns: dictionary equivalent as dict_swh only with its keys converted. """ + def convert_hashes_bytes(v): """v is supposedly a hash as bytes, returns it converted in hex. """ if isinstance(v, bytes): return hashutil.hash_to_hex(v) return v def convert_bytes(v): """v is supposedly a bytes string, decode as utf-8. FIXME: Improve decoding policy. If not utf-8, break! """ if isinstance(v, bytes): - return v.decode('utf-8') + return v.decode("utf-8") return v def convert_date(v): """ Args: v (dict or datatime): either: - a dict with three keys: - timestamp (dict or integer timestamp) - offset - negative_utc - or, a datetime We convert it to a human-readable string """ if not v: return v if isinstance(v, datetime.datetime): return v.isoformat() - tz = datetime.timezone(datetime.timedelta(minutes=v['offset'])) - swh_timestamp = v['timestamp'] + tz = datetime.timezone(datetime.timedelta(minutes=v["offset"])) + swh_timestamp = v["timestamp"] if isinstance(swh_timestamp, dict): - date = datetime.datetime.fromtimestamp( - swh_timestamp['seconds'], tz=tz) + date = datetime.datetime.fromtimestamp(swh_timestamp["seconds"], tz=tz) else: - date = datetime.datetime.fromtimestamp( - swh_timestamp, tz=tz) + date = datetime.datetime.fromtimestamp(swh_timestamp, tz=tz) datestr = date.isoformat() - if v['offset'] == 0 and v['negative_utc']: + if v["offset"] == 0 and v["negative_utc"]: # remove the rightmost + and replace it with a - - return '-'.join(datestr.rsplit('+', 1)) + return "-".join(datestr.rsplit("+", 1)) return datestr if not dict_swh: return dict_swh new_dict = {} for key, value in dict_swh.items(): if key in blacklist or (key in removables_if_empty and not value): continue if key in dates: new_dict[key] = convert_date(value) elif key in convert: new_dict[key] = convert_fn(value) elif isinstance(value, dict): - new_dict[key] = from_swh(value, - hashess=hashess, bytess=bytess, - dates=dates, blacklist=blacklist, - removables_if_empty=removables_if_empty, - empty_dict=empty_dict, - empty_list=empty_list, - convert=convert, - convert_fn=convert_fn) + new_dict[key] = from_swh( + value, + hashess=hashess, + bytess=bytess, + dates=dates, + blacklist=blacklist, + removables_if_empty=removables_if_empty, + empty_dict=empty_dict, + empty_list=empty_list, + convert=convert, + convert_fn=convert_fn, + ) elif key in hashess: new_dict[key] = fmap(convert_hashes_bytes, value) elif key in bytess: try: new_dict[key] = fmap(convert_bytes, value) except UnicodeDecodeError: - if 'decoding_failures' not in new_dict: - new_dict['decoding_failures'] = [key] + if "decoding_failures" not in new_dict: + new_dict["decoding_failures"] = [key] else: - new_dict['decoding_failures'].append(key) + new_dict["decoding_failures"].append(key) new_dict[key] = fmap(decode_with_escape, value) elif key in empty_dict and not value: new_dict[key] = {} elif key in empty_list and not value: new_dict[key] = [] else: new_dict[key] = value _group_checksums(new_dict) return new_dict def from_origin(origin): """Convert from a swh origin to an origin dictionary. """ return from_swh(origin) def from_release(release): """Convert from a swh release to a json serializable release dictionary. Args: release (dict): dictionary with keys: - id: identifier of the revision (sha1 in bytes) - revision: identifier of the revision the release points to (sha1 in bytes) comment: release's comment message (bytes) name: release's name (string) author: release's author identifier (swh's id) synthetic: the synthetic property (boolean) Returns: dict: Release dictionary with the following keys: - id: hexadecimal sha1 (string) - revision: hexadecimal sha1 (string) - comment: release's comment message (string) - name: release's name (string) - author: release's author identifier (swh's id) - synthetic: the synthetic property (boolean) """ return from_swh( release, - hashess={'id', 'target'}, - bytess={'message', 'name', 'fullname', 'email'}, - dates={'date'}, + hashess={"id", "target"}, + bytess={"message", "name", "fullname", "email"}, + dates={"date"}, ) class SWHMetadataEncoder(json.JSONEncoder): """Special json encoder for metadata field which can contain bytes encoded value. """ + def default(self, obj): if isinstance(obj, bytes): try: - return obj.decode('utf-8') + return obj.decode("utf-8") except UnicodeDecodeError: # fallback to binary representation to avoid display errors return repr(obj) # Let the base class default method raise the TypeError return json.JSONEncoder.default(self, obj) def convert_revision_metadata(metadata): """Convert json specific dict to a json serializable one. """ if not metadata: return {} return json.loads(json.dumps(metadata, cls=SWHMetadataEncoder)) def from_revision(revision): """Convert from a swh revision to a json serializable revision dictionary. Args: revision (dict): dict with keys: - id: identifier of the revision (sha1 in bytes) - directory: identifier of the directory the revision points to (sha1 in bytes) - author_name, author_email: author's revision name and email - committer_name, committer_email: committer's revision name and email - message: revision's message - date, date_offset: revision's author date - committer_date, committer_date_offset: revision's commit date - parents: list of parents for such revision - synthetic: revision's property nature - type: revision's type (git, tar or dsc at the moment) - metadata: if the revision is synthetic, this can reference dynamic properties. Returns: dict: Revision dictionary with the same keys as inputs, except: - sha1s are in hexadecimal strings (id, directory) - bytes are decoded in string (author_name, committer_name, author_email, committer_email) Remaining keys are left as is """ - revision = from_swh(revision, - hashess={'id', 'directory', 'parents', 'children'}, - bytess={'name', 'fullname', 'email'}, - convert={'metadata'}, - convert_fn=convert_revision_metadata, - dates={'date', 'committer_date'}) + revision = from_swh( + revision, + hashess={"id", "directory", "parents", "children"}, + bytess={"name", "fullname", "email"}, + convert={"metadata"}, + convert_fn=convert_revision_metadata, + dates={"date", "committer_date"}, + ) if revision: - if 'parents' in revision: - revision['merge'] = len(revision['parents']) > 1 - if 'message' in revision: + if "parents" in revision: + revision["merge"] = len(revision["parents"]) > 1 + if "message" in revision: try: - revision['message'] = revision['message'].decode('utf-8') + revision["message"] = revision["message"].decode("utf-8") except UnicodeDecodeError: - revision['message_decoding_failed'] = True - revision['message'] = None + revision["message_decoding_failed"] = True + revision["message"] = None return revision def from_content(content): """Convert swh content to serializable content dictionary. """ - return from_swh(content, - hashess={'sha1', 'sha1_git', 'sha256', 'blake2s256'}, - blacklist={'ctime'}, - convert={'status'}, - convert_fn=lambda v: 'absent' if v == 'hidden' else v) + return from_swh( + content, + hashess={"sha1", "sha1_git", "sha256", "blake2s256"}, + blacklist={"ctime"}, + convert={"status"}, + convert_fn=lambda v: "absent" if v == "hidden" else v, + ) def from_person(person): """Convert swh person to serializable person dictionary. """ - return from_swh(person, - bytess={'name', 'fullname', 'email'}) + return from_swh(person, bytess={"name", "fullname", "email"}) def from_origin_visit(visit): """Convert swh origin_visit to serializable origin_visit dictionary. """ - ov = from_swh(visit, - hashess={'target', 'snapshot'}, - bytess={'branch'}, - dates={'date'}, - empty_dict={'metadata'}) + ov = from_swh( + visit, + hashess={"target", "snapshot"}, + bytess={"branch"}, + dates={"date"}, + empty_dict={"metadata"}, + ) return ov def from_snapshot(snapshot): """Convert swh snapshot to serializable snapshot dictionary. """ - sv = from_swh(snapshot, - hashess={'id', 'target'}, - bytess={'next_branch'}) - - if sv and 'branches' in sv: - sv['branches'] = { - decode_with_escape(k): v - for k, v in sv['branches'].items() - } - for k, v in snapshot['branches'].items(): + sv = from_swh(snapshot, hashess={"id", "target"}, bytess={"next_branch"}) + + if sv and "branches" in sv: + sv["branches"] = {decode_with_escape(k): v for k, v in sv["branches"].items()} + for k, v in snapshot["branches"].items(): # alias target existing branch names, not a sha1 - if v and v['target_type'] == 'alias': + if v and v["target_type"] == "alias": branch = decode_with_escape(k) - target = decode_with_escape(v['target']) - sv['branches'][branch]['target'] = target + target = decode_with_escape(v["target"]) + sv["branches"][branch]["target"] = target return sv def from_directory_entry(dir_entry): """Convert swh directory to serializable directory dictionary. """ - return from_swh(dir_entry, - hashess={'dir_id', 'sha1_git', 'sha1', 'sha256', - 'blake2s256', 'target'}, - bytess={'name'}, - removables_if_empty={ - 'sha1', 'sha1_git', 'sha256', 'blake2s256', 'status'}, - convert={'status'}, - convert_fn=lambda v: 'absent' if v == 'hidden' else v) + return from_swh( + dir_entry, + hashess={"dir_id", "sha1_git", "sha1", "sha256", "blake2s256", "target"}, + bytess={"name"}, + removables_if_empty={"sha1", "sha1_git", "sha256", "blake2s256", "status"}, + convert={"status"}, + convert_fn=lambda v: "absent" if v == "hidden" else v, + ) def from_filetype(content_entry): """Convert swh content to serializable dictionary containing keys 'id', 'encoding', and 'mimetype'. """ - return from_swh(content_entry, - hashess={'id'}) + return from_swh(content_entry, hashess={"id"}) diff --git a/swh/web/common/exc.py b/swh/web/common/exc.py index fc94316d..178dd346 100644 --- a/swh/web/common/exc.py +++ b/swh/web/common/exc.py @@ -1,135 +1,150 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import traceback from django.http import HttpResponse from django.shortcuts import render from django.utils.safestring import mark_safe from django.utils.html import escape import sentry_sdk from swh.web.config import get_config class BadInputExc(ValueError): """Wrong request to the api. Example: Asking a content with the wrong identifier format. """ + pass class NotFoundExc(Exception): """Good request to the api but no result were found. Example: Asking a content with the right identifier format but that content does not exist. """ + pass class ForbiddenExc(Exception): """Good request to the api, forbidden result to return due to enforce policy. Example: Asking for a raw content which exists but whose mimetype is not text. """ + pass class LargePayloadExc(Exception): """The input size is too large. Example: Asking to resolve 10000 persistent identifier when the limit is 1000. """ + pass http_status_code_message = { - 400: 'Bad Request', - 401: 'Unauthorized', - 403: 'Access Denied', - 404: 'Resource not found', - 413: 'Payload Too Large', - 500: 'Internal Server Error', - 501: 'Not Implemented', - 502: 'Bad Gateway', - 503: 'Service unavailable' + 400: "Bad Request", + 401: "Unauthorized", + 403: "Access Denied", + 404: "Resource not found", + 413: "Payload Too Large", + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", + 503: "Service unavailable", } def _generate_error_page(request, error_code, error_description): - return render(request, 'error.html', - {'error_code': error_code, - 'error_message': http_status_code_message[error_code], - 'error_description': mark_safe(error_description)}, - status=error_code) + return render( + request, + "error.html", + { + "error_code": error_code, + "error_message": http_status_code_message[error_code], + "error_description": mark_safe(error_description), + }, + status=error_code, + ) def swh_handle400(request, exception=None): """ Custom Django HTTP error 400 handler for swh-web. """ - error_description = ('The server cannot process the request to %s due to ' - 'something that is perceived to be a client error.' % - escape(request.META['PATH_INFO'])) + error_description = ( + "The server cannot process the request to %s due to " + "something that is perceived to be a client error." + % escape(request.META["PATH_INFO"]) + ) return _generate_error_page(request, 400, error_description) def swh_handle403(request, exception=None): """ Custom Django HTTP error 403 handler for swh-web. """ - error_description = ('The resource %s requires an authentication.' % - escape(request.META['PATH_INFO'])) + error_description = "The resource %s requires an authentication." % escape( + request.META["PATH_INFO"] + ) return _generate_error_page(request, 403, error_description) def swh_handle404(request, exception=None): """ Custom Django HTTP error 404 handler for swh-web. """ - error_description = ('The resource %s could not be found on the server.' % - escape(request.META['PATH_INFO'])) + error_description = "The resource %s could not be found on the server." % escape( + request.META["PATH_INFO"] + ) return _generate_error_page(request, 404, error_description) def swh_handle500(request): """ Custom Django HTTP error 500 handler for swh-web. """ - error_description = ('An unexpected condition was encountered when ' - 'requesting resource %s.' % - escape(request.META['PATH_INFO'])) + error_description = ( + "An unexpected condition was encountered when " + "requesting resource %s." % escape(request.META["PATH_INFO"]) + ) return _generate_error_page(request, 500, error_description) def handle_view_exception(request, exc, html_response=True): """ Function used to generate an error page when an exception was raised inside a swh-web browse view. """ sentry_sdk.capture_exception(exc) error_code = 500 - error_description = '%s: %s' % (type(exc).__name__, str(exc)) - if get_config()['debug']: + error_description = "%s: %s" % (type(exc).__name__, str(exc)) + if get_config()["debug"]: error_description = traceback.format_exc() if isinstance(exc, BadInputExc): error_code = 400 if isinstance(exc, ForbiddenExc): error_code = 403 if isinstance(exc, NotFoundExc): error_code = 404 if html_response: return _generate_error_page(request, error_code, error_description) else: - return HttpResponse(error_description, content_type='text/plain', - status=error_code) + return HttpResponse( + error_description, content_type="text/plain", status=error_code + ) diff --git a/swh/web/common/highlightjs.py b/swh/web/common/highlightjs.py index e6b8e124..47fb6a1b 100644 --- a/swh/web/common/highlightjs.py +++ b/swh/web/common/highlightjs.py @@ -1,361 +1,519 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from typing import Dict -from pygments.lexers import ( - get_all_lexers, - get_lexer_for_filename -) +from pygments.lexers import get_all_lexers, get_lexer_for_filename import sentry_sdk # set of languages ids that can be highlighted # by highlight.js library -_hljs_languages = set([ - '1c', 'abnf', 'accesslog', 'actionscript', 'ada', 'angelscript', 'apache', - 'applescript', 'arcade', 'arduino', 'armasm', 'asciidoc', 'aspectj', - 'autohotkey', 'autoit', 'avrasm', 'awk', 'axapta', 'bash', 'basic', 'bnf', - 'brainfuck', 'cal', 'capnproto', 'ceylon', 'clean', 'clojure', - 'clojure-repl', 'cmake', 'coffeescript', 'coq', 'cos', 'cpp', 'crmsh', - 'crystal', 'cs', 'csp', 'css', 'd', 'dart', 'delphi', 'diff', 'django', - 'dns', 'dockerfile', 'dos', 'dsconfig', 'dts', 'dust', 'ebnf', 'elixir', - 'elm', 'erb', 'erlang', 'erlang-repl', 'excel', 'fix', 'flix', 'fortran', - 'fsharp', 'gams', 'gauss', 'gcode', 'gherkin', 'glsl', 'gml', 'go', - 'golo', 'gradle', 'groovy', 'haml', 'handlebars', 'haskell', 'haxe', - 'hsp', 'htmlbars', 'http', 'hy', 'inform7', 'ini', 'irpf90', 'isbl', - 'java', 'javascript', 'jboss-cli', 'json', 'julia', 'julia-repl', - 'kotlin', 'lasso', 'ldif', 'leaf', 'less', 'lisp', 'livecodeserver', - 'livescript', 'llvm', 'lsl', 'lua', 'makefile', 'markdown', 'mathematica', - 'matlab', 'maxima', 'mel', 'mercury', 'mipsasm', 'mizar', 'mojolicious', - 'monkey', 'moonscript', 'n1ql', 'nginx', 'nimrod', 'nix', 'nsis', - 'objectivec', 'ocaml', 'openscad', 'oxygene', 'parser3', 'perl', 'pf', - 'pgsql', 'php', 'plaintext', 'pony', 'powershell', 'processing', - 'profile', 'prolog', 'properties', 'protobuf', 'puppet', 'purebasic', - 'python', 'q', 'qml', 'r', 'reasonml', 'rib', 'roboconf', 'routeros', - 'rsl', 'ruby', 'ruleslanguage', 'rust', 'sas', 'scala', 'scheme', - 'scilab', 'scss', 'shell', 'smali', 'smalltalk', 'sml', 'sqf', 'sql', - 'stan', 'stata', 'step21', 'stylus', 'subunit', 'swift', 'taggerscript', - 'tap', 'tcl', 'tex', 'thrift', 'tp', 'twig', 'typescript', 'vala', - 'vbnet', 'vbscript', 'vbscript-html', 'verilog', 'vhdl', 'vim', 'x86asm', - 'xl', 'xml', 'xquery', 'yaml', 'zephir' -]) +_hljs_languages = set( + [ + "1c", + "abnf", + "accesslog", + "actionscript", + "ada", + "angelscript", + "apache", + "applescript", + "arcade", + "arduino", + "armasm", + "asciidoc", + "aspectj", + "autohotkey", + "autoit", + "avrasm", + "awk", + "axapta", + "bash", + "basic", + "bnf", + "brainfuck", + "cal", + "capnproto", + "ceylon", + "clean", + "clojure", + "clojure-repl", + "cmake", + "coffeescript", + "coq", + "cos", + "cpp", + "crmsh", + "crystal", + "cs", + "csp", + "css", + "d", + "dart", + "delphi", + "diff", + "django", + "dns", + "dockerfile", + "dos", + "dsconfig", + "dts", + "dust", + "ebnf", + "elixir", + "elm", + "erb", + "erlang", + "erlang-repl", + "excel", + "fix", + "flix", + "fortran", + "fsharp", + "gams", + "gauss", + "gcode", + "gherkin", + "glsl", + "gml", + "go", + "golo", + "gradle", + "groovy", + "haml", + "handlebars", + "haskell", + "haxe", + "hsp", + "htmlbars", + "http", + "hy", + "inform7", + "ini", + "irpf90", + "isbl", + "java", + "javascript", + "jboss-cli", + "json", + "julia", + "julia-repl", + "kotlin", + "lasso", + "ldif", + "leaf", + "less", + "lisp", + "livecodeserver", + "livescript", + "llvm", + "lsl", + "lua", + "makefile", + "markdown", + "mathematica", + "matlab", + "maxima", + "mel", + "mercury", + "mipsasm", + "mizar", + "mojolicious", + "monkey", + "moonscript", + "n1ql", + "nginx", + "nimrod", + "nix", + "nsis", + "objectivec", + "ocaml", + "openscad", + "oxygene", + "parser3", + "perl", + "pf", + "pgsql", + "php", + "plaintext", + "pony", + "powershell", + "processing", + "profile", + "prolog", + "properties", + "protobuf", + "puppet", + "purebasic", + "python", + "q", + "qml", + "r", + "reasonml", + "rib", + "roboconf", + "routeros", + "rsl", + "ruby", + "ruleslanguage", + "rust", + "sas", + "scala", + "scheme", + "scilab", + "scss", + "shell", + "smali", + "smalltalk", + "sml", + "sqf", + "sql", + "stan", + "stata", + "step21", + "stylus", + "subunit", + "swift", + "taggerscript", + "tap", + "tcl", + "tex", + "thrift", + "tp", + "twig", + "typescript", + "vala", + "vbnet", + "vbscript", + "vbscript-html", + "verilog", + "vhdl", + "vim", + "x86asm", + "xl", + "xml", + "xquery", + "yaml", + "zephir", + ] +) # languages aliases defined in highlight.js _hljs_languages_aliases = { - 'ado': 'stata', - 'adoc': 'asciidoc', - 'ahk': 'autohotkey', - 'aj': 'aspectj', - 'apacheconf': 'apache', - 'arm': 'armasm', - 'as': 'actionscript', - 'asc': 'asciidoc', - 'atom': 'xml', - 'bas': 'basic', - 'bat': 'dos', - 'bf': 'brainfuck', - 'bind': 'dns', - 'bsl': '1c', - 'c-al': 'cal', - 'c': 'cpp', - 'c++': 'cpp', - 'capnp': 'capnproto', - 'cc': 'cpp', - 'clj': 'clojure', - 'cls': 'cos', - 'cmake.in': 'cmake', - 'cmd': 'dos', - 'coffee': 'coffeescript', - 'console': 'shell', - 'cr': 'crystal', - 'craftcms': 'twig', - 'crm': 'crmsh', - 'csharp': 'cs', - 'cson': 'coffeescript', - 'dcl': 'clean', - 'dfm': 'delphi', - 'do': 'stata', - 'docker': 'dockerfile', - 'dpr': 'delphi', - 'dst': 'dust', - 'dtsi': 'dts', - 'ep': 'mojolicious', - 'erl': 'erlang', - 'ex': 'elixir', - 'exs': 'elixir', - 'f90': 'fortran', - 'f95': 'fortran', - 'feature': 'gherkin', - 'freepascal': 'delphi', - 'fs': 'fsharp', - 'fsx': 'fsharp', - 'gemspec': 'ruby', - 'GML': 'gml', - 'gms': 'gams', - 'golang': 'go', - 'graph': 'roboconf', - 'gss': 'gauss', - 'gyp': 'python', - 'h': 'cpp', - 'h++': 'cpp', - 'hbs': 'handlebars', - 'hpp': 'cpp', - 'hs': 'haskell', - 'html': 'xml', - 'html.handlebars': 'handlebars', - 'html.hbs': 'handlebars', - 'https': 'http', - 'hx': 'haxe', - 'hylang': 'hy', - 'i7': 'inform7', - 'i7x': 'inform7', - 'iced': 'coffeescript', - 'icl': 'clean', - 'ino': 'arduino', - 'instances': 'roboconf', - 'ipynb': 'json', - 'irb': 'ruby', - 'jinja': 'django', - 'js': 'javascript', - 'jsp': 'java', - 'jsx': 'javascript', - 'k': 'q', - 'kdb': 'q', - 'kt': 'kotlin', - 'lassoscript': 'lasso', - 'lazarus': 'delphi', - 'lc': 'livecode', - 'lfm': 'delphi', - 'll': 'llvm', - 'lpr': 'delphi', - 'ls': 'livescript', - 'm': 'matlab', - 'mak': 'makefile', - 'md': 'markdown', - 'mikrotik': 'routeros', - 'mips': 'mipsasm', - 'mk': 'monkey', - 'mkd': 'markdown', - 'mkdown': 'markdown', - 'ml': 'ocaml', - 'mli': 'ocaml', - 'mm': 'objectivec', - 'mma': 'mathematica', - 'moo': 'mercury', - 'moon': 'moonscript', - 'nav': 'cal', - 'nb': 'mathematica', - 'nc': 'gcode', - 'nginxconf': 'nginx', - 'ni': 'inform7', - 'nim': 'nimrod', - 'nixos': 'nix', - 'nsi': 'nsis', - 'obj-c': 'objectivec', - 'objc': 'objectivec', - 'osascript': 'applescript', - 'osl': 'rsl', - 'p': 'parser3', - 'p21': 'step21', - 'pas': 'delphi', - 'pascal': 'delphi', - 'patch': 'diff', - 'pb': 'purebasic', - 'pbi': 'purebasic', - 'pcmk': 'crmsh', - 'pde': 'processing', - 'pf.conf': 'pf', - 'php3': 'php', - 'php4': 'php', - 'php5': 'php', - 'php6': 'php', - 'php7': 'php', - 'pl': 'perl', - 'plist': 'xml', - 'pm': 'perl', - 'podspec': 'ruby', - 'postgres': 'pgsql', - 'postgresql': 'pgsql', - 'pp': 'puppet', - 'proto': 'protobuf', - 'ps': 'powershell', - 'ps1': 'powershell', - 'psd1': 'powershell', - 'psm1': 'powershell', - 'py': 'python', - 'qt': 'qml', - 'rb': 'ruby', - 're': 'reasonml', - 'rei': 'reasonml', - 'rs': 'rust', - 'rsc': 'routeros', - 'rss': 'xml', - 'rst': 'nohighlight', - 's': 'armasm', - 'SAS': 'sas', - 'scad': 'openscad', - 'sci': 'scilab', - 'scm': 'scheme', - 'sh': 'bash', - 'sig': 'sml', - 'sl': 'rsl', - 'st': 'smalltalk', - 'step': 'step21', - 'stp': 'step21', - 'styl': 'stylus', - 'sv': 'verilog', - 'svh': 'verilog', - 'tao': 'xl', - 'thor': 'ruby', - 'tk': 'tcl', - 'toml': 'ini', - 'ts': 'typescript', - 'txt': 'nohighlight', - 'v': 'coq', - 'vb': 'vbnet', - 'vbs': 'vbscript', - 'vhd': 'vhdl', - 'wildfly-cli': 'jboss-cli', - 'wl': 'mathematica', - 'wls': 'mathematica', - 'xhtml': 'xml', - 'xjb': 'xml', - 'xls': 'excel', - 'xlsx': 'excel', - 'xpath': 'xquery', - 'xpo': 'axapta', - 'xpp': 'axapta', - 'xq': 'xquery', - 'xqy': 'xquery', - 'xsd': 'xml', - 'xsl': 'xml', - 'YAML': 'yaml', - 'yml': 'yaml', - 'zep': 'zephir', - 'zone': 'dns', - 'zsh': 'bash' + "ado": "stata", + "adoc": "asciidoc", + "ahk": "autohotkey", + "aj": "aspectj", + "apacheconf": "apache", + "arm": "armasm", + "as": "actionscript", + "asc": "asciidoc", + "atom": "xml", + "bas": "basic", + "bat": "dos", + "bf": "brainfuck", + "bind": "dns", + "bsl": "1c", + "c-al": "cal", + "c": "cpp", + "c++": "cpp", + "capnp": "capnproto", + "cc": "cpp", + "clj": "clojure", + "cls": "cos", + "cmake.in": "cmake", + "cmd": "dos", + "coffee": "coffeescript", + "console": "shell", + "cr": "crystal", + "craftcms": "twig", + "crm": "crmsh", + "csharp": "cs", + "cson": "coffeescript", + "dcl": "clean", + "dfm": "delphi", + "do": "stata", + "docker": "dockerfile", + "dpr": "delphi", + "dst": "dust", + "dtsi": "dts", + "ep": "mojolicious", + "erl": "erlang", + "ex": "elixir", + "exs": "elixir", + "f90": "fortran", + "f95": "fortran", + "feature": "gherkin", + "freepascal": "delphi", + "fs": "fsharp", + "fsx": "fsharp", + "gemspec": "ruby", + "GML": "gml", + "gms": "gams", + "golang": "go", + "graph": "roboconf", + "gss": "gauss", + "gyp": "python", + "h": "cpp", + "h++": "cpp", + "hbs": "handlebars", + "hpp": "cpp", + "hs": "haskell", + "html": "xml", + "html.handlebars": "handlebars", + "html.hbs": "handlebars", + "https": "http", + "hx": "haxe", + "hylang": "hy", + "i7": "inform7", + "i7x": "inform7", + "iced": "coffeescript", + "icl": "clean", + "ino": "arduino", + "instances": "roboconf", + "ipynb": "json", + "irb": "ruby", + "jinja": "django", + "js": "javascript", + "jsp": "java", + "jsx": "javascript", + "k": "q", + "kdb": "q", + "kt": "kotlin", + "lassoscript": "lasso", + "lazarus": "delphi", + "lc": "livecode", + "lfm": "delphi", + "ll": "llvm", + "lpr": "delphi", + "ls": "livescript", + "m": "matlab", + "mak": "makefile", + "md": "markdown", + "mikrotik": "routeros", + "mips": "mipsasm", + "mk": "monkey", + "mkd": "markdown", + "mkdown": "markdown", + "ml": "ocaml", + "mli": "ocaml", + "mm": "objectivec", + "mma": "mathematica", + "moo": "mercury", + "moon": "moonscript", + "nav": "cal", + "nb": "mathematica", + "nc": "gcode", + "nginxconf": "nginx", + "ni": "inform7", + "nim": "nimrod", + "nixos": "nix", + "nsi": "nsis", + "obj-c": "objectivec", + "objc": "objectivec", + "osascript": "applescript", + "osl": "rsl", + "p": "parser3", + "p21": "step21", + "pas": "delphi", + "pascal": "delphi", + "patch": "diff", + "pb": "purebasic", + "pbi": "purebasic", + "pcmk": "crmsh", + "pde": "processing", + "pf.conf": "pf", + "php3": "php", + "php4": "php", + "php5": "php", + "php6": "php", + "php7": "php", + "pl": "perl", + "plist": "xml", + "pm": "perl", + "podspec": "ruby", + "postgres": "pgsql", + "postgresql": "pgsql", + "pp": "puppet", + "proto": "protobuf", + "ps": "powershell", + "ps1": "powershell", + "psd1": "powershell", + "psm1": "powershell", + "py": "python", + "qt": "qml", + "rb": "ruby", + "re": "reasonml", + "rei": "reasonml", + "rs": "rust", + "rsc": "routeros", + "rss": "xml", + "rst": "nohighlight", + "s": "armasm", + "SAS": "sas", + "scad": "openscad", + "sci": "scilab", + "scm": "scheme", + "sh": "bash", + "sig": "sml", + "sl": "rsl", + "st": "smalltalk", + "step": "step21", + "stp": "step21", + "styl": "stylus", + "sv": "verilog", + "svh": "verilog", + "tao": "xl", + "thor": "ruby", + "tk": "tcl", + "toml": "ini", + "ts": "typescript", + "txt": "nohighlight", + "v": "coq", + "vb": "vbnet", + "vbs": "vbscript", + "vhd": "vhdl", + "wildfly-cli": "jboss-cli", + "wl": "mathematica", + "wls": "mathematica", + "xhtml": "xml", + "xjb": "xml", + "xls": "excel", + "xlsx": "excel", + "xpath": "xquery", + "xpo": "axapta", + "xpp": "axapta", + "xq": "xquery", + "xqy": "xquery", + "xsd": "xml", + "xsl": "xml", + "YAML": "yaml", + "yml": "yaml", + "zep": "zephir", + "zone": "dns", + "zsh": "bash", } # dictionary mapping pygment lexers to hljs languages _pygments_lexer_to_hljs_language = {} # type: Dict[str, str] # dictionary mapping mime types to hljs languages _mime_type_to_hljs_language = { - 'text/x-c': 'cpp', - 'text/x-c++': 'cpp', - 'text/x-msdos-batch': 'dos', - 'text/x-lisp': 'lisp', - 'text/x-shellscript': 'bash', + "text/x-c": "cpp", + "text/x-c++": "cpp", + "text/x-msdos-batch": "dos", + "text/x-lisp": "lisp", + "text/x-shellscript": "bash", } # dictionary mapping filenames to hljs languages _filename_to_hljs_language = { - 'cmakelists.txt': 'cmake', - '.htaccess': 'apache', - 'httpd.conf': 'apache', - 'access.log': 'accesslog', - 'nginx.log': 'accesslog', - 'resolv.conf': 'dns', - 'dockerfile': 'docker', - 'nginx.conf': 'nginx', - 'pf.conf': 'pf' + "cmakelists.txt": "cmake", + ".htaccess": "apache", + "httpd.conf": "apache", + "access.log": "accesslog", + "nginx.log": "accesslog", + "resolv.conf": "dns", + "dockerfile": "docker", + "nginx.conf": "nginx", + "pf.conf": "pf", } # function to fill the above dictionaries def _init_pygments_to_hljs_map(): if len(_pygments_lexer_to_hljs_language) == 0: for lexer in get_all_lexers(): lexer_name = lexer[0] lang_aliases = lexer[1] lang_mime_types = lexer[3] lang = None for lang_alias in lang_aliases: if lang_alias in _hljs_languages: lang = lang_alias _pygments_lexer_to_hljs_language[lexer_name] = lang_alias break if lang: for lang_mime_type in lang_mime_types: _mime_type_to_hljs_language[lang_mime_type] = lang def get_hljs_language_from_filename(filename): """Function that tries to associate a language supported by highlight.js from a filename. Args: filename: input filename Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if filename: filename_lower = filename.lower() if filename_lower in _filename_to_hljs_language: return _filename_to_hljs_language[filename_lower] if filename_lower in _hljs_languages: return filename_lower - exts = filename_lower.split('.') + exts = filename_lower.split(".") # check if file extension matches an hljs language # also handle .ext.in cases for ext in reversed(exts[-2:]): if ext in _hljs_languages: return ext if ext in _hljs_languages_aliases: return _hljs_languages_aliases[ext] # otherwise use Pygments language database lexer = None # try to find a Pygment lexer try: lexer = get_lexer_for_filename(filename) except Exception as exc: sentry_sdk.capture_exception(exc) # if there is a correspondence between the lexer and an hljs # language, return it if lexer and lexer.name in _pygments_lexer_to_hljs_language: return _pygments_lexer_to_hljs_language[lexer.name] # otherwise, try to find a match between the file extensions # associated to the lexer and the hljs language aliases if lexer: - exts = [ext.replace('*.', '') for ext in lexer.filenames] + exts = [ext.replace("*.", "") for ext in lexer.filenames] for ext in exts: if ext in _hljs_languages_aliases: return _hljs_languages_aliases[ext] return None def get_hljs_language_from_mime_type(mime_type): """Function that tries to associate a language supported by highlight.js from a mime type. Args: mime_type: input mime type Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if mime_type and mime_type in _mime_type_to_hljs_language: return _mime_type_to_hljs_language[mime_type] return None @functools.lru_cache() def get_supported_languages(): """ Return the list of programming languages that can be highlighted using the highlight.js library. Returns: List[str]: the list of supported languages """ return sorted(list(_hljs_languages)) diff --git a/swh/web/common/identifiers.py b/swh/web/common/identifiers.py index 3679958e..c0df94da 100644 --- a/swh/web/common/identifiers.py +++ b/swh/web/common/identifiers.py @@ -1,180 +1,197 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict, Iterable, List, Optional from typing_extensions import TypedDict from django.http import QueryDict from swh.model.exceptions import ValidationError from swh.model.hashutil import hash_to_bytes from swh.model.identifiers import ( - persistent_identifier, parse_persistent_identifier, - CONTENT, DIRECTORY, ORIGIN, RELEASE, REVISION, SNAPSHOT, - PersistentId + persistent_identifier, + parse_persistent_identifier, + CONTENT, + DIRECTORY, + ORIGIN, + RELEASE, + REVISION, + SNAPSHOT, + PersistentId, ) from swh.web.common.exc import BadInputExc from swh.web.common.typing import QueryParameters from swh.web.common.utils import reverse -def get_swh_persistent_id(object_type: str, object_id: str, - scheme_version: int = 1) -> str: +def get_swh_persistent_id( + object_type: str, object_id: str, scheme_version: int = 1 +) -> str: """ Returns the persistent identifier for a swh object based on: * the object type * the object id * the swh identifiers scheme version Args: object_type: the swh object type (content/directory/release/revision/snapshot) object_id: the swh object id (hexadecimal representation of its hash value) scheme_version: the scheme version of the swh persistent identifiers Returns: the swh object persistent identifier Raises: BadInputExc: if the provided parameters do not enable to generate a valid identifier """ try: swh_id = persistent_identifier(object_type, object_id, scheme_version) except ValidationError as e: - raise BadInputExc('Invalid object (%s) for swh persistent id. %s' % - (object_id, e)) + raise BadInputExc( + "Invalid object (%s) for swh persistent id. %s" % (object_id, e) + ) else: return swh_id -ResolvedPersistentId = TypedDict('ResolvedPersistentId', { - 'swh_id_parsed': PersistentId, - 'browse_url': Optional[str] -}) +ResolvedPersistentId = TypedDict( + "ResolvedPersistentId", {"swh_id_parsed": PersistentId, "browse_url": Optional[str]} +) -def resolve_swh_persistent_id(swh_id: str, - query_params: Optional[QueryParameters] = None - ) -> ResolvedPersistentId: +def resolve_swh_persistent_id( + swh_id: str, query_params: Optional[QueryParameters] = None +) -> ResolvedPersistentId: """ Try to resolve a Software Heritage persistent id into an url for browsing the targeted object. Args: swh_id: a Software Heritage persistent identifier query_params: optional dict filled with query parameters to append to the browse url Returns: a dict with the following keys: * **swh_id_parsed**: the parsed identifier * **browse_url**: the url for browsing the targeted object """ swh_id_parsed = get_persistent_identifier(swh_id) object_type = swh_id_parsed.object_type object_id = swh_id_parsed.object_id browse_url = None - query_dict = QueryDict('', mutable=True) + query_dict = QueryDict("", mutable=True) if query_params and len(query_params) > 0: for k in sorted(query_params.keys()): query_dict[k] = query_params[k] - if 'origin' in swh_id_parsed.metadata: - query_dict['origin'] = swh_id_parsed.metadata['origin'] + if "origin" in swh_id_parsed.metadata: + query_dict["origin"] = swh_id_parsed.metadata["origin"] if object_type == CONTENT: - query_string = 'sha1_git:' + object_id - fragment = '' - if 'lines' in swh_id_parsed.metadata: - lines = swh_id_parsed.metadata['lines'].split('-') - fragment += '#L' + lines[0] + query_string = "sha1_git:" + object_id + fragment = "" + if "lines" in swh_id_parsed.metadata: + lines = swh_id_parsed.metadata["lines"].split("-") + fragment += "#L" + lines[0] if len(lines) > 1: - fragment += '-L' + lines[1] - browse_url = reverse('browse-content', - url_args={'query_string': query_string}, - query_params=query_dict) + fragment + fragment += "-L" + lines[1] + browse_url = ( + reverse( + "browse-content", + url_args={"query_string": query_string}, + query_params=query_dict, + ) + + fragment + ) elif object_type == DIRECTORY: - browse_url = reverse('browse-directory', - url_args={'sha1_git': object_id}, - query_params=query_dict) + browse_url = reverse( + "browse-directory", + url_args={"sha1_git": object_id}, + query_params=query_dict, + ) elif object_type == RELEASE: - browse_url = reverse('browse-release', - url_args={'sha1_git': object_id}, - query_params=query_dict) + browse_url = reverse( + "browse-release", url_args={"sha1_git": object_id}, query_params=query_dict + ) elif object_type == REVISION: - browse_url = reverse('browse-revision', - url_args={'sha1_git': object_id}, - query_params=query_dict) + browse_url = reverse( + "browse-revision", url_args={"sha1_git": object_id}, query_params=query_dict + ) elif object_type == SNAPSHOT: - browse_url = reverse('browse-snapshot', - url_args={'snapshot_id': object_id}, - query_params=query_dict) + browse_url = reverse( + "browse-snapshot", + url_args={"snapshot_id": object_id}, + query_params=query_dict, + ) elif object_type == ORIGIN: - raise BadInputExc(('Origin PIDs (Persistent Identifiers) are not ' - 'publicly resolvable because they are for ' - 'internal usage only')) + raise BadInputExc( + ( + "Origin PIDs (Persistent Identifiers) are not " + "publicly resolvable because they are for " + "internal usage only" + ) + ) - return { - 'swh_id_parsed': swh_id_parsed, - 'browse_url': browse_url - } + return {"swh_id_parsed": swh_id_parsed, "browse_url": browse_url} def get_persistent_identifier(persistent_id: str) -> PersistentId: """Check if a persistent identifier is valid. Args: persistent_id: A string representing a Software Heritage persistent identifier. Raises: BadInputExc: if the provided persistent identifier can not be parsed. Return: A persistent identifier object. """ try: pid_object = parse_persistent_identifier(persistent_id) except ValidationError as ve: - raise BadInputExc('Error when parsing identifier: %s' % - ' '.join(ve.messages)) + raise BadInputExc("Error when parsing identifier: %s" % " ".join(ve.messages)) else: return pid_object -def group_swh_persistent_identifiers(persistent_ids: Iterable[PersistentId] - ) -> Dict[str, List[bytes]]: +def group_swh_persistent_identifiers( + persistent_ids: Iterable[PersistentId], +) -> Dict[str, List[bytes]]: """ Groups many Software Heritage persistent identifiers into a dictionary depending on their type. Args: persistent_ids: an iterable of Software Heritage persistent identifier objects Returns: A dictionary with: keys: persistent identifier types values: persistent identifiers id """ pids_by_type: Dict[str, List[bytes]] = { CONTENT: [], DIRECTORY: [], REVISION: [], RELEASE: [], - SNAPSHOT: [] + SNAPSHOT: [], } for pid in persistent_ids: obj_id = pid.object_id obj_type = pid.object_type pids_by_type[obj_type].append(hash_to_bytes(obj_id)) return pids_by_type diff --git a/swh/web/common/middlewares.py b/swh/web/common/middlewares.py index 7fceaf59..84fd1e6a 100644 --- a/swh/web/common/middlewares.py +++ b/swh/web/common/middlewares.py @@ -1,71 +1,71 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from htmlmin import minify import sentry_sdk from swh.web.common.utils import prettify_html class HtmlPrettifyMiddleware(object): """ Django middleware for prettifying generated HTML in development mode. """ def __init__(self, get_response): self.get_response = get_response def __call__(self, request): response = self.get_response(request) - if 'text/html' in response.get('Content-Type', ''): - if hasattr(response, 'content'): + if "text/html" in response.get("Content-Type", ""): + if hasattr(response, "content"): content = response.content response.content = prettify_html(content) - elif hasattr(response, 'streaming_content'): - content = b''.join(response.streaming_content) + elif hasattr(response, "streaming_content"): + content = b"".join(response.streaming_content) response.streaming_content = prettify_html(content) return response class HtmlMinifyMiddleware(object): """ Django middleware for minifying generated HTML in production mode. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): response = self.get_response(request) - if 'text/html' in response.get('Content-Type', ''): + if "text/html" in response.get("Content-Type", ""): try: - minified_html = minify(response.content.decode('utf-8')) - response.content = minified_html.encode('utf-8') + minified_html = minify(response.content.decode("utf-8")) + response.content = minified_html.encode("utf-8") except Exception as exc: sentry_sdk.capture_exception(exc) return response class ThrottlingHeadersMiddleware(object): """ Django middleware for inserting rate limiting related headers in HTTP response. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): resp = self.get_response(request) - if 'RateLimit-Limit' in request.META: - resp['X-RateLimit-Limit'] = request.META['RateLimit-Limit'] - if 'RateLimit-Remaining' in request.META: - resp['X-RateLimit-Remaining'] = request.META['RateLimit-Remaining'] - if 'RateLimit-Reset' in request.META: - resp['X-RateLimit-Reset'] = request.META['RateLimit-Reset'] + if "RateLimit-Limit" in request.META: + resp["X-RateLimit-Limit"] = request.META["RateLimit-Limit"] + if "RateLimit-Remaining" in request.META: + resp["X-RateLimit-Remaining"] = request.META["RateLimit-Remaining"] + if "RateLimit-Reset" in request.META: + resp["X-RateLimit-Reset"] = request.META["RateLimit-Reset"] return resp diff --git a/swh/web/common/migrations/0001_initial.py b/swh/web/common/migrations/0001_initial.py index b81f050f..ac4877e6 100644 --- a/swh/web/common/migrations/0001_initial.py +++ b/swh/web/common/migrations/0001_initial.py @@ -1,72 +1,89 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -# flake8: noqa - from __future__ import unicode_literals from django.db import migrations, models _authorized_origins = [ - 'https://github.com/', - 'https://gitlab.com/', - 'https://bitbucket.org/', - 'https://git.code.sf.net/', - 'http://git.code.sf.net/', - 'https://hg.code.sf.net/', - 'http://hg.code.sf.net/', - 'https://svn.code.sf.net/', - 'http://svn.code.sf.net/' + "https://github.com/", + "https://gitlab.com/", + "https://bitbucket.org/", + "https://git.code.sf.net/", + "http://git.code.sf.net/", + "https://hg.code.sf.net/", + "http://hg.code.sf.net/", + "https://svn.code.sf.net/", + "http://svn.code.sf.net/", ] def _populate_save_authorized_origins(apps, schema_editor): - SaveAuthorizedOrigin = apps.get_model('swh.web.common', 'SaveAuthorizedOrigin') + SaveAuthorizedOrigin = apps.get_model("swh.web.common", "SaveAuthorizedOrigin") for origin_url in _authorized_origins: SaveAuthorizedOrigin.objects.create(url=origin_url) class Migration(migrations.Migration): initial = True operations = [ migrations.CreateModel( - name='SaveAuthorizedOrigin', + name="SaveAuthorizedOrigin", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('url', models.CharField(max_length=200)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("url", models.CharField(max_length=200)), ], - options={ - 'db_table': 'save_authorized_origin', - }, + options={"db_table": "save_authorized_origin",}, ), migrations.CreateModel( - name='SaveOriginRequest', + name="SaveOriginRequest", fields=[ - ('id', models.BigAutoField(primary_key=True, serialize=False)), - ('request_date', models.DateTimeField(auto_now_add=True)), - ('origin_type', models.CharField(max_length=200)), - ('origin_url', models.CharField(max_length=200)), - ('status', models.TextField(choices=[('accepted', 'accepted'), ('rejected', 'rejected'), ('pending', 'pending')], default='pending')), - ('loading_task_id', models.IntegerField(default=-1)), + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("request_date", models.DateTimeField(auto_now_add=True)), + ("origin_type", models.CharField(max_length=200)), + ("origin_url", models.CharField(max_length=200)), + ( + "status", + models.TextField( + choices=[ + ("accepted", "accepted"), + ("rejected", "rejected"), + ("pending", "pending"), + ], + default="pending", + ), + ), + ("loading_task_id", models.IntegerField(default=-1)), ], - options={ - 'db_table': 'save_origin_request', - 'ordering': ['-id'], - }, + options={"db_table": "save_origin_request", "ordering": ["-id"],}, ), migrations.CreateModel( - name='SaveUnauthorizedOrigin', + name="SaveUnauthorizedOrigin", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('url', models.CharField(max_length=200)), + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("url", models.CharField(max_length=200)), ], - options={ - 'db_table': 'save_unauthorized_origin', - }, + options={"db_table": "save_unauthorized_origin",}, ), - migrations.RunPython(_populate_save_authorized_origins) + migrations.RunPython(_populate_save_authorized_origins), ] diff --git a/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py b/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py index 0847505e..73729ae4 100644 --- a/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py +++ b/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py @@ -1,23 +1,23 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('swh.web.common', '0001_initial'), + ("swh.web.common", "0001_initial"), ] operations = [ migrations.AddField( - model_name='saveoriginrequest', - name='visit_date', + model_name="saveoriginrequest", + name="visit_date", field=models.DateTimeField(null=True), ), ] diff --git a/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py b/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py index 5b32752a..ed2d2878 100644 --- a/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py +++ b/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py @@ -1,44 +1,52 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -# flake8: noqa - from __future__ import unicode_literals from django.db import migrations, models from swh.web.config import scheduler + def _remove_archived_tasks_with_no_saved_status(apps, schema_editor): """ Scheduler tasks are archived on a regular basis so their completion state could not be known anymore as previous to this migration, the loading task status was not stored in the database. So remove the rows associated to already archived tasks as the loading status can not be retrieved anymore. """ - SaveOriginRequest = apps.get_model('swh.web.common', 'SaveOriginRequest') + SaveOriginRequest = apps.get_model("swh.web.common", "SaveOriginRequest") no_saved_status_tasks = [] for sor in SaveOriginRequest.objects.all(): tasks = scheduler().get_tasks([sor.loading_task_id]) if not tasks: no_saved_status_tasks.append(sor.loading_task_id) SaveOriginRequest.objects.filter(loading_task_id__in=no_saved_status_tasks).delete() class Migration(migrations.Migration): dependencies = [ - ('swh.web.common', '0002_saveoriginrequest_visit_date'), + ("swh.web.common", "0002_saveoriginrequest_visit_date"), ] operations = [ migrations.AddField( - model_name='saveoriginrequest', - name='loading_task_status', - field=models.TextField(choices=[('not created', 'not created'), ('not yet scheduled', 'not yet scheduled'), ('scheduled', 'scheduled'), ('succeed', 'succeed'), ('failed', 'failed')], default='not created'), + model_name="saveoriginrequest", + name="loading_task_status", + field=models.TextField( + choices=[ + ("not created", "not created"), + ("not yet scheduled", "not yet scheduled"), + ("scheduled", "scheduled"), + ("succeed", "succeed"), + ("failed", "failed"), + ], + default="not created", + ), ), - migrations.RunPython(_remove_archived_tasks_with_no_saved_status) + migrations.RunPython(_remove_archived_tasks_with_no_saved_status), ] diff --git a/swh/web/common/migrations/0004_auto_20190204_1324.py b/swh/web/common/migrations/0004_auto_20190204_1324.py index 6be22fef..23513812 100644 --- a/swh/web/common/migrations/0004_auto_20190204_1324.py +++ b/swh/web/common/migrations/0004_auto_20190204_1324.py @@ -1,25 +1,33 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -# flake8: noqa - from __future__ import unicode_literals from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('swh.web.common', '0003_saveoriginrequest_loading_task_status'), + ("swh.web.common", "0003_saveoriginrequest_loading_task_status"), ] operations = [ migrations.AlterField( - model_name='saveoriginrequest', - name='loading_task_status', - field=models.TextField(choices=[('not created', 'not created'), ('not yet scheduled', 'not yet scheduled'), ('scheduled', 'scheduled'), ('succeed', 'succeed'), ('failed', 'failed'), ('running', 'running')], default='not created'), + model_name="saveoriginrequest", + name="loading_task_status", + field=models.TextField( + choices=[ + ("not created", "not created"), + ("not yet scheduled", "not yet scheduled"), + ("scheduled", "scheduled"), + ("succeed", "succeed"), + ("failed", "failed"), + ("running", "running"), + ], + default="not created", + ), ), ] diff --git a/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py b/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py index e4f78f80..e64f3093 100644 --- a/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py +++ b/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py @@ -1,28 +1,25 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations from swh.web.common.models import SaveAuthorizedOrigin def _remove_duplicated_urls_in_authorized_list(apps, schema_editor): sao = SaveAuthorizedOrigin.objects - for url in sao.values_list('url', flat=True).distinct(): - sao.filter(pk__in=sao.filter( - url=url).values_list('id', flat=True)[1:]).delete() + for url in sao.values_list("url", flat=True).distinct(): + sao.filter(pk__in=sao.filter(url=url).values_list("id", flat=True)[1:]).delete() class Migration(migrations.Migration): dependencies = [ - ('swh.web.common', '0004_auto_20190204_1324'), + ("swh.web.common", "0004_auto_20190204_1324"), ] - operations = [ - migrations.RunPython(_remove_duplicated_urls_in_authorized_list) - ] + operations = [migrations.RunPython(_remove_duplicated_urls_in_authorized_list)] diff --git a/swh/web/common/migrations/0006_rename_origin_type.py b/swh/web/common/migrations/0006_rename_origin_type.py index 017b9e9e..7474fa7f 100644 --- a/swh/web/common/migrations/0006_rename_origin_type.py +++ b/swh/web/common/migrations/0006_rename_origin_type.py @@ -1,23 +1,23 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ('swh.web.common', '0005_remove_duplicated_authorized_origins'), + ("swh.web.common", "0005_remove_duplicated_authorized_origins"), ] operations = [ migrations.RenameField( - model_name='saveoriginrequest', - old_name='origin_type', - new_name='visit_type', + model_name="saveoriginrequest", + old_name="origin_type", + new_name="visit_type", ), ] diff --git a/swh/web/common/models.py b/swh/web/common/models.py index f0b66c5f..b0e54cd6 100644 --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -1,92 +1,99 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import models class SaveAuthorizedOrigin(models.Model): """ Model table holding origin urls authorized to be loaded into the archive. """ + url = models.CharField(max_length=200, null=False) class Meta: - app_label = 'swh.web.common' - db_table = 'save_authorized_origin' + app_label = "swh.web.common" + db_table = "save_authorized_origin" def __str__(self): return self.url class SaveUnauthorizedOrigin(models.Model): """ Model table holding origin urls not authorized to be loaded into the archive. """ + url = models.CharField(max_length=200, null=False) class Meta: - app_label = 'swh.web.common' - db_table = 'save_unauthorized_origin' + app_label = "swh.web.common" + db_table = "save_unauthorized_origin" def __str__(self): return self.url -SAVE_REQUEST_ACCEPTED = 'accepted' -SAVE_REQUEST_REJECTED = 'rejected' -SAVE_REQUEST_PENDING = 'pending' +SAVE_REQUEST_ACCEPTED = "accepted" +SAVE_REQUEST_REJECTED = "rejected" +SAVE_REQUEST_PENDING = "pending" SAVE_REQUEST_STATUS = [ (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_ACCEPTED), (SAVE_REQUEST_REJECTED, SAVE_REQUEST_REJECTED), - (SAVE_REQUEST_PENDING, SAVE_REQUEST_PENDING) + (SAVE_REQUEST_PENDING, SAVE_REQUEST_PENDING), ] -SAVE_TASK_NOT_CREATED = 'not created' -SAVE_TASK_NOT_YET_SCHEDULED = 'not yet scheduled' -SAVE_TASK_SCHEDULED = 'scheduled' -SAVE_TASK_SUCCEED = 'succeed' -SAVE_TASK_FAILED = 'failed' -SAVE_TASK_RUNNING = 'running' +SAVE_TASK_NOT_CREATED = "not created" +SAVE_TASK_NOT_YET_SCHEDULED = "not yet scheduled" +SAVE_TASK_SCHEDULED = "scheduled" +SAVE_TASK_SUCCEED = "succeed" +SAVE_TASK_FAILED = "failed" +SAVE_TASK_RUNNING = "running" SAVE_TASK_STATUS = [ (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_CREATED), (SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED), (SAVE_TASK_SCHEDULED, SAVE_TASK_SCHEDULED), (SAVE_TASK_SUCCEED, SAVE_TASK_SUCCEED), (SAVE_TASK_FAILED, SAVE_TASK_FAILED), - (SAVE_TASK_RUNNING, SAVE_TASK_RUNNING) + (SAVE_TASK_RUNNING, SAVE_TASK_RUNNING), ] class SaveOriginRequest(models.Model): """ Model table holding all the save origin requests issued by users. """ + id = models.BigAutoField(primary_key=True) request_date = models.DateTimeField(auto_now_add=True) visit_type = models.CharField(max_length=200, null=False) origin_url = models.CharField(max_length=200, null=False) - status = models.TextField(choices=SAVE_REQUEST_STATUS, - default=SAVE_REQUEST_PENDING) + status = models.TextField(choices=SAVE_REQUEST_STATUS, default=SAVE_REQUEST_PENDING) loading_task_id = models.IntegerField(default=-1) visit_date = models.DateTimeField(null=True) - loading_task_status = models.TextField(choices=SAVE_TASK_STATUS, - default=SAVE_TASK_NOT_CREATED) + loading_task_status = models.TextField( + choices=SAVE_TASK_STATUS, default=SAVE_TASK_NOT_CREATED + ) class Meta: - app_label = 'swh.web.common' - db_table = 'save_origin_request' - ordering = ['-id'] + app_label = "swh.web.common" + db_table = "save_origin_request" + ordering = ["-id"] def __str__(self): - return str({'id': self.id, - 'request_date': self.request_date, - 'visit_type': self.visit_type, - 'origin_url': self.origin_url, - 'status': self.status, - 'loading_task_id': self.loading_task_id, - 'visit_date': self.visit_date}) + return str( + { + "id": self.id, + "request_date": self.request_date, + "visit_type": self.visit_type, + "origin_url": self.origin_url, + "status": self.status, + "loading_task_id": self.loading_task_id, + "visit_date": self.visit_date, + } + ) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py index bfdf8896..c8173474 100644 --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -1,585 +1,596 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from bisect import bisect_right from datetime import datetime, timezone, timedelta from itertools import product import json import logging from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError from django.core.validators import URLValidator from django.utils.html import escape from prometheus_client import Gauge import requests import sentry_sdk from swh.web import config from swh.web.common import service from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.common.models import ( - SaveUnauthorizedOrigin, SaveAuthorizedOrigin, SaveOriginRequest, - SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, - SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, - SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING, - SAVE_TASK_NOT_CREATED + SaveUnauthorizedOrigin, + SaveAuthorizedOrigin, + SaveOriginRequest, + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING, + SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, + SAVE_TASK_SUCCEED, + SAVE_TASK_FAILED, + SAVE_TASK_RUNNING, + SAVE_TASK_NOT_CREATED, ) from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import parse_timestamp, SWH_WEB_METRICS_REGISTRY from swh.scheduler.utils import create_oneshot_task_dict scheduler = config.scheduler() logger = logging.getLogger(__name__) def get_origin_save_authorized_urls(): """ Get the list of origin url prefixes authorized to be immediately loaded into the archive (whitelist). Returns: list: The list of authorized origin url prefix """ - return [origin.url - for origin in SaveAuthorizedOrigin.objects.all()] + return [origin.url for origin in SaveAuthorizedOrigin.objects.all()] def get_origin_save_unauthorized_urls(): """ Get the list of origin url prefixes forbidden to be loaded into the archive (blacklist). Returns: list: the list of unauthorized origin url prefix """ - return [origin.url - for origin in SaveUnauthorizedOrigin.objects.all()] + return [origin.url for origin in SaveUnauthorizedOrigin.objects.all()] def can_save_origin(origin_url): """ Check if a software origin can be saved into the archive. Based on the origin url, the save request will be either: * immediately accepted if the url is whitelisted * rejected if the url is blacklisted * put in pending state for manual review otherwise Args: origin_url (str): the software origin url to check Returns: str: the origin save request status, either **accepted**, **rejected** or **pending** """ # origin url may be blacklisted for url_prefix in get_origin_save_unauthorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_REJECTED # if the origin url is in the white list, it can be immediately saved for url_prefix in get_origin_save_authorized_urls(): if origin_url.startswith(url_prefix): return SAVE_REQUEST_ACCEPTED # otherwise, the origin url needs to be manually verified return SAVE_REQUEST_PENDING # map visit type to scheduler task # TODO: do not hardcode the task name here (T1157) -_visit_type_task = { - 'git': 'load-git', - 'hg': 'load-hg', - 'svn': 'load-svn' -} +_visit_type_task = {"git": "load-git", "hg": "load-hg", "svn": "load-svn"} # map scheduler task status to origin save status _save_task_status = { - 'next_run_not_scheduled': SAVE_TASK_NOT_YET_SCHEDULED, - 'next_run_scheduled': SAVE_TASK_SCHEDULED, - 'completed': SAVE_TASK_SUCCEED, - 'disabled': SAVE_TASK_FAILED + "next_run_not_scheduled": SAVE_TASK_NOT_YET_SCHEDULED, + "next_run_scheduled": SAVE_TASK_SCHEDULED, + "completed": SAVE_TASK_SUCCEED, + "disabled": SAVE_TASK_FAILED, } def get_savable_visit_types(): return sorted(list(_visit_type_task.keys())) def _check_visit_type_savable(visit_type): """ Get the list of visit types that can be performed through a save request. Returns: list: the list of saveable visit types """ - allowed_visit_types = ', '.join(get_savable_visit_types()) + allowed_visit_types = ", ".join(get_savable_visit_types()) if visit_type not in _visit_type_task: - raise BadInputExc('Visit of type %s can not be saved! ' - 'Allowed types are the following: %s' % - (visit_type, allowed_visit_types)) + raise BadInputExc( + "Visit of type %s can not be saved! " + "Allowed types are the following: %s" % (visit_type, allowed_visit_types) + ) -_validate_url = URLValidator(schemes=['http', 'https', 'svn', 'git']) +_validate_url = URLValidator(schemes=["http", "https", "svn", "git"]) def _check_origin_url_valid(origin_url): try: _validate_url(origin_url) except ValidationError: - raise BadInputExc('The provided origin url (%s) is not valid!' % - escape(origin_url)) + raise BadInputExc( + "The provided origin url (%s) is not valid!" % escape(origin_url) + ) def _get_visit_info_for_save_request(save_request): visit_date = None visit_status = None try: - origin = {'url': save_request.origin_url} + origin = {"url": save_request.origin_url} origin_info = service.lookup_origin(origin) origin_visits = get_origin_visits(origin_info) - visit_dates = [parse_timestamp(v['date']) - for v in origin_visits] + visit_dates = [parse_timestamp(v["date"]) for v in origin_visits] i = bisect_right(visit_dates, save_request.request_date) if i != len(visit_dates): visit_date = visit_dates[i] - visit_status = origin_visits[i]['status'] - if origin_visits[i]['status'] == 'ongoing': + visit_status = origin_visits[i]["status"] + if origin_visits[i]["status"] == "ongoing": visit_date = None except Exception as exc: sentry_sdk.capture_exception(exc) return visit_date, visit_status def _check_visit_update_status(save_request, save_task_status): visit_date, visit_status = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date # visit has been performed, mark the saving task as succeed if visit_date and visit_status is not None: save_task_status = SAVE_TASK_SUCCEED - elif visit_status == 'ongoing': + elif visit_status == "ongoing": save_task_status = SAVE_TASK_RUNNING else: time_now = datetime.now(tz=timezone.utc) time_delta = time_now - save_request.request_date # consider the task as failed if it is still in scheduled state # 30 days after its submission if time_delta.days > 30: save_task_status = SAVE_TASK_FAILED return visit_date, save_task_status def _save_request_dict(save_request, task=None): must_save = False visit_date = save_request.visit_date # save task still in scheduler db if task: - save_task_status = _save_task_status[task['status']] + save_task_status = _save_task_status[task["status"]] # Consider request from which a visit date has already been found # as succeeded to avoid retrieving it again if save_task_status == SAVE_TASK_SCHEDULED and visit_date: save_task_status = SAVE_TASK_SUCCEED - if save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEED) \ - and not visit_date: + if save_task_status in (SAVE_TASK_FAILED, SAVE_TASK_SUCCEED) and not visit_date: visit_date, _ = _get_visit_info_for_save_request(save_request) save_request.visit_date = visit_date must_save = True # Check tasks still marked as scheduled / not yet scheduled - if save_task_status in (SAVE_TASK_SCHEDULED, - SAVE_TASK_NOT_YET_SCHEDULED): + if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( - save_request, save_task_status) + save_request, save_task_status + ) # save task may have been archived else: save_task_status = save_request.loading_task_status - if save_task_status in (SAVE_TASK_SCHEDULED, - SAVE_TASK_NOT_YET_SCHEDULED): + if save_task_status in (SAVE_TASK_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED): visit_date, save_task_status = _check_visit_update_status( - save_request, save_task_status) + save_request, save_task_status + ) else: save_task_status = save_request.loading_task_status if save_request.loading_task_status != save_task_status: save_request.loading_task_status = save_task_status must_save = True if must_save: save_request.save() - return {'id': save_request.id, - 'visit_type': save_request.visit_type, - 'origin_url': save_request.origin_url, - 'save_request_date': save_request.request_date.isoformat(), - 'save_request_status': save_request.status, - 'save_task_status': save_task_status, - 'visit_date': visit_date.isoformat() if visit_date else None} + return { + "id": save_request.id, + "visit_type": save_request.visit_type, + "origin_url": save_request.origin_url, + "save_request_date": save_request.request_date.isoformat(), + "save_request_status": save_request.status, + "save_task_status": save_task_status, + "visit_date": visit_date.isoformat() if visit_date else None, + } def create_save_origin_request(visit_type, origin_url): """ Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. First, some checks are performed to see if the visit type and origin url are valid but also if the the save request can be accepted. If those checks passed, the loading task is then created. Otherwise, the save request is put in pending or rejected state. All the submitted save requests are logged into the swh-web database to keep track of them. Args: visit_type (str): the type of visit to perform (currently only ``git`` but ``svn`` and ``hg`` will soon be available) origin_url (str): the url of the origin to save Raises: BadInputExc: the visit type or origin url is invalid ForbiddenExc: the provided origin url is blacklisted Returns: dict: A dict describing the save request with the following keys: * **visit_type**: the type of visit to perform * **origin_url**: the url of the origin * **save_request_date**: the date the request was submitted * **save_request_status**: the request status, either **accepted**, **rejected** or **pending** * **save_task_status**: the origin loading task status, either **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** """ _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) save_request_status = can_save_origin(origin_url) task = None # if the origin save request is accepted, create a scheduler # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority kwargs = { - 'priority': 'high', - 'url': origin_url, + "priority": "high", + "url": origin_url, } sor = None # get list of previously sumitted save requests - current_sors = \ - list(SaveOriginRequest.objects.filter(visit_type=visit_type, - origin_url=origin_url)) + current_sors = list( + SaveOriginRequest.objects.filter( + visit_type=visit_type, origin_url=origin_url + ) + ) can_create_task = False # if no save requests previously submitted, create the scheduler task if not current_sors: can_create_task = True else: # get the latest submitted save request sor = current_sors[0] # if it was in pending state, we need to create the scheduler task # and update the save request info in the database if sor.status == SAVE_REQUEST_PENDING: can_create_task = True # a task has already been created to load the origin elif sor.loading_task_id != -1: # get the scheduler task and its status tasks = scheduler.get_tasks([sor.loading_task_id]) task = tasks[0] if tasks else None - task_status = _save_request_dict(sor, task)['save_task_status'] + task_status = _save_request_dict(sor, task)["save_task_status"] # create a new scheduler task only if the previous one has been # already executed - if task_status == SAVE_TASK_FAILED or \ - task_status == SAVE_TASK_SUCCEED: + if task_status == SAVE_TASK_FAILED or task_status == SAVE_TASK_SUCCEED: can_create_task = True sor = None else: can_create_task = False if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict( - _visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted if sor: sor.status = SAVE_REQUEST_ACCEPTED - sor.loading_task_id = task['id'] + sor.loading_task_id = task["id"] sor.save() else: - sor = SaveOriginRequest.objects.create(visit_type=visit_type, - origin_url=origin_url, - status=save_request_status, # noqa - loading_task_id=task['id']) # noqa + sor = SaveOriginRequest.objects.create( + visit_type=visit_type, + origin_url=origin_url, + status=save_request_status, + loading_task_id=task["id"], + ) # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, # no need to add it to the database in that case try: - sor = SaveOriginRequest.objects.get(visit_type=visit_type, - origin_url=origin_url, - status=save_request_status) + sor = SaveOriginRequest.objects.get( + visit_type=visit_type, origin_url=origin_url, status=save_request_status + ) # if not add it to the database except ObjectDoesNotExist: - sor = SaveOriginRequest.objects.create(visit_type=visit_type, - origin_url=origin_url, - status=save_request_status) + sor = SaveOriginRequest.objects.create( + visit_type=visit_type, origin_url=origin_url, status=save_request_status + ) # origin can not be saved as its url is blacklisted, # log the request to the database anyway else: - sor = SaveOriginRequest.objects.create(visit_type=visit_type, - origin_url=origin_url, - status=save_request_status) + sor = SaveOriginRequest.objects.create( + visit_type=visit_type, origin_url=origin_url, status=save_request_status + ) if save_request_status == SAVE_REQUEST_REJECTED: - raise ForbiddenExc(('The "save code now" request has been rejected ' - 'because the provided origin url is blacklisted.')) + raise ForbiddenExc( + ( + 'The "save code now" request has been rejected ' + "because the provided origin url is blacklisted." + ) + ) return _save_request_dict(sor, task) def get_save_origin_requests_from_queryset(requests_queryset): """ Get all save requests from a SaveOriginRequest queryset. Args: requests_queryset (django.db.models.QuerySet): input SaveOriginRequest queryset Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ task_ids = [] for sor in requests_queryset: task_ids.append(sor.loading_task_id) save_requests = [] if task_ids: tasks = scheduler.get_tasks(task_ids) - tasks = {task['id']: task for task in tasks} + tasks = {task["id"]: task for task in tasks} for sor in requests_queryset: sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id)) save_requests.append(sr_dict) return save_requests def get_save_origin_requests(visit_type, origin_url): """ Get all save requests for a given software origin. Args: visit_type (str): the type of visit origin_url (str): the url of the origin Raises: BadInputExc: the visit type or origin url is invalid swh.web.common.exc.NotFoundExc: no save requests can be found for the given origin Returns: list: A list of save origin requests dict as described in :func:`swh.web.common.origin_save.create_save_origin_request` """ _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) - sors = SaveOriginRequest.objects.filter(visit_type=visit_type, - origin_url=origin_url) + sors = SaveOriginRequest.objects.filter( + visit_type=visit_type, origin_url=origin_url + ) if sors.count() == 0: - raise NotFoundExc(('No save requests found for visit of type ' - '%s on origin with url %s.') - % (visit_type, origin_url)) + raise NotFoundExc( + ("No save requests found for visit of type " "%s on origin with url %s.") + % (visit_type, origin_url) + ) return get_save_origin_requests_from_queryset(sors) def get_save_origin_task_info(save_request_id): """ Get detailed information about an accepted save origin request and its associated loading task. If the associated loading task info is archived and removed from the scheduler database, returns an empty dictionary. Args: save_request_id (int): identifier of a save origin request Returns: dict: A dictionary with the following keys: - **type**: loading task type - **arguments**: loading task arguments - **id**: loading task database identifier - **backend_id**: loading task celery identifier - **scheduled**: loading task scheduling date - **ended**: loading task termination date - **status**: loading task execution status Depending on the availability of the task logs in the elasticsearch cluster of Software Heritage, the returned dictionary may also contain the following keys: - **name**: associated celery task name - **message**: relevant log message from task execution - **duration**: task execution time (only if it succeeded) - **worker**: name of the worker that executed the task """ try: save_request = SaveOriginRequest.objects.get(id=save_request_id) except ObjectDoesNotExist: return {} task = scheduler.get_tasks([save_request.loading_task_id]) task = task[0] if task else None if task is None: return {} - task_run = scheduler.get_task_runs([task['id']]) + task_run = scheduler.get_task_runs([task["id"]]) task_run = task_run[0] if task_run else None if task_run is None: return {} - task_run['type'] = task['type'] - task_run['arguments'] = task['arguments'] - task_run['id'] = task_run['task'] - del task_run['task'] - del task_run['metadata'] - del task_run['started'] - - es_workers_index_url = config.get_config()['es_workers_index_url'] + task_run["type"] = task["type"] + task_run["arguments"] = task["arguments"] + task_run["id"] = task_run["task"] + del task_run["task"] + del task_run["metadata"] + del task_run["started"] + + es_workers_index_url = config.get_config()["es_workers_index_url"] if not es_workers_index_url: return task_run - es_workers_index_url += '/_search' + es_workers_index_url += "/_search" if save_request.visit_date: min_ts = save_request.visit_date max_ts = min_ts + timedelta(days=7) else: min_ts = save_request.request_date max_ts = min_ts + timedelta(days=30) min_ts = int(min_ts.timestamp()) * 1000 max_ts = int(max_ts.timestamp()) * 1000 - save_task_status = _save_task_status[task['status']] - priority = '3' if save_task_status == SAVE_TASK_FAILED else '6' + save_task_status = _save_task_status[task["status"]] + priority = "3" if save_task_status == SAVE_TASK_FAILED else "6" query = { - 'bool': { - 'must': [ - { - 'match_phrase': { - 'priority': { - 'query': priority - } - } - }, + "bool": { + "must": [ + {"match_phrase": {"priority": {"query": priority}}}, + {"match_phrase": {"swh_task_id": {"query": task_run["backend_id"]}}}, { - 'match_phrase': { - 'swh_task_id': { - 'query': task_run['backend_id'] + "range": { + "@timestamp": { + "gte": min_ts, + "lte": max_ts, + "format": "epoch_millis", } } }, - { - 'range': { - '@timestamp': { - 'gte': min_ts, - 'lte': max_ts, - 'format': 'epoch_millis' - } - } - } ] } } try: - response = requests.post(es_workers_index_url, - json={'query': query, - 'sort': ['@timestamp']}, - timeout=30) + response = requests.post( + es_workers_index_url, + json={"query": query, "sort": ["@timestamp"]}, + timeout=30, + ) results = json.loads(response.text) - if results['hits']['total']['value'] >= 1: - task_run_info = results['hits']['hits'][-1]['_source'] - if 'swh_logging_args_runtime' in task_run_info: - duration = task_run_info['swh_logging_args_runtime'] - task_run['duration'] = duration - if 'message' in task_run_info: - task_run['message'] = task_run_info['message'] - if 'swh_logging_args_name' in task_run_info: - task_run['name'] = task_run_info['swh_logging_args_name'] - elif 'swh_task_name' in task_run_info: - task_run['name'] = task_run_info['swh_task_name'] - if 'hostname' in task_run_info: - task_run['worker'] = task_run_info['hostname'] - elif 'host' in task_run_info: - task_run['worker'] = task_run_info['host'] + if results["hits"]["total"]["value"] >= 1: + task_run_info = results["hits"]["hits"][-1]["_source"] + if "swh_logging_args_runtime" in task_run_info: + duration = task_run_info["swh_logging_args_runtime"] + task_run["duration"] = duration + if "message" in task_run_info: + task_run["message"] = task_run_info["message"] + if "swh_logging_args_name" in task_run_info: + task_run["name"] = task_run_info["swh_logging_args_name"] + elif "swh_task_name" in task_run_info: + task_run["name"] = task_run_info["swh_task_name"] + if "hostname" in task_run_info: + task_run["worker"] = task_run_info["hostname"] + elif "host" in task_run_info: + task_run["worker"] = task_run_info["host"] except Exception as exc: - logger.warning('Request to Elasticsearch failed\n%s', exc) + logger.warning("Request to Elasticsearch failed\n%s", exc) sentry_sdk.capture_exception(exc) return task_run -SUBMITTED_SAVE_REQUESTS_METRIC = 'swh_web_submitted_save_requests' +SUBMITTED_SAVE_REQUESTS_METRIC = "swh_web_submitted_save_requests" _submitted_save_requests_gauge = Gauge( name=SUBMITTED_SAVE_REQUESTS_METRIC, - documentation='Number of submitted origin save requests', - labelnames=['status', 'visit_type'], - registry=SWH_WEB_METRICS_REGISTRY) + documentation="Number of submitted origin save requests", + labelnames=["status", "visit_type"], + registry=SWH_WEB_METRICS_REGISTRY, +) -ACCEPTED_SAVE_REQUESTS_METRIC = 'swh_web_accepted_save_requests' +ACCEPTED_SAVE_REQUESTS_METRIC = "swh_web_accepted_save_requests" _accepted_save_requests_gauge = Gauge( name=ACCEPTED_SAVE_REQUESTS_METRIC, - documentation='Number of accepted origin save requests', - labelnames=['load_task_status', 'visit_type'], - registry=SWH_WEB_METRICS_REGISTRY) + documentation="Number of accepted origin save requests", + labelnames=["load_task_status", "visit_type"], + registry=SWH_WEB_METRICS_REGISTRY, +) def compute_save_requests_metrics(): """Compute a couple of Prometheus metrics related to origin save requests""" - request_statuses = (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, - SAVE_REQUEST_PENDING) - - load_task_statuses = (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, - SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, - SAVE_TASK_FAILED, SAVE_TASK_RUNNING) + request_statuses = ( + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING, + ) + + load_task_statuses = ( + SAVE_TASK_NOT_CREATED, + SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, + SAVE_TASK_SUCCEED, + SAVE_TASK_FAILED, + SAVE_TASK_RUNNING, + ) visit_types = get_savable_visit_types() labels_set = product(request_statuses, visit_types) for labels in labels_set: _submitted_save_requests_gauge.labels(*labels).set(0) labels_set = product(load_task_statuses, visit_types) for labels in labels_set: _accepted_save_requests_gauge.labels(*labels).set(0) for sor in SaveOriginRequest.objects.all(): if sor.status == SAVE_REQUEST_ACCEPTED: _accepted_save_requests_gauge.labels( - load_task_status=sor.loading_task_status, - visit_type=sor.visit_type).inc() + load_task_status=sor.loading_task_status, visit_type=sor.visit_type + ).inc() _submitted_save_requests_gauge.labels( - status=sor.status, visit_type=sor.visit_type).inc() + status=sor.status, visit_type=sor.visit_type + ).inc() diff --git a/swh/web/common/origin_visits.py b/swh/web/common/origin_visits.py index 7072d8ce..7de92729 100644 --- a/swh/web/common/origin_visits.py +++ b/swh/web/common/origin_visits.py @@ -1,159 +1,175 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import math from django.core.cache import cache from swh.web.common.exc import NotFoundExc from swh.web.common.utils import parse_timestamp def get_origin_visits(origin_info): """Function that returns the list of visits for a swh origin. That list is put in cache in order to speedup the navigation in the swh web browse ui. Args: origin_info (dict): dict describing the origin to fetch visits from Returns: list: A list of dict describing the origin visits with the following keys: * **date**: UTC visit date in ISO format, * **origin**: the origin url * **status**: the visit status, either **full**, **partial** or **ongoing** * **visit**: the visit id * **type**: the visit type Raises: swh.web.common.exc.NotFoundExc: if the origin is not found """ from swh.web.common import service - if 'url' in origin_info: - origin_url = origin_info['url'] + if "url" in origin_info: + origin_url = origin_info["url"] else: - origin_url = service.lookup_origin(origin_info)['url'] + origin_url = service.lookup_origin(origin_info)["url"] - cache_entry_id = 'origin_visits_%s' % origin_url + cache_entry_id = "origin_visits_%s" % origin_url cache_entry = cache.get(cache_entry_id) if cache_entry: - last_visit = cache_entry[-1]['visit'] - new_visits = list(service.lookup_origin_visits(origin_url, - last_visit=last_visit)) + last_visit = cache_entry[-1]["visit"] + new_visits = list( + service.lookup_origin_visits(origin_url, last_visit=last_visit) + ) if not new_visits: last_snp = service.lookup_latest_origin_snapshot(origin_url) - if not last_snp or last_snp['id'] == cache_entry[-1]['snapshot']: + if not last_snp or last_snp["id"] == cache_entry[-1]["snapshot"]: return cache_entry origin_visits = [] per_page = service.MAX_LIMIT last_visit = None while 1: - visits = list(service.lookup_origin_visits(origin_url, - last_visit=last_visit, - per_page=per_page)) + visits = list( + service.lookup_origin_visits( + origin_url, last_visit=last_visit, per_page=per_page + ) + ) origin_visits += visits if len(visits) < per_page: break else: if not last_visit: last_visit = per_page else: last_visit += per_page def _visit_sort_key(visit): - ts = parse_timestamp(visit['date']).timestamp() - return ts + (float(visit['visit']) / 10e3) + ts = parse_timestamp(visit["date"]).timestamp() + return ts + (float(visit["visit"]) / 10e3) for v in origin_visits: - if 'metadata' in v: - del v['metadata'] - origin_visits = [dict(t) for t in set([tuple(d.items()) - for d in origin_visits])] + if "metadata" in v: + del v["metadata"] + origin_visits = [dict(t) for t in set([tuple(d.items()) for d in origin_visits])] origin_visits = sorted(origin_visits, key=lambda v: _visit_sort_key(v)) cache.set(cache_entry_id, origin_visits) return origin_visits -def get_origin_visit(origin_info, visit_ts=None, visit_id=None, - snapshot_id=None): +def get_origin_visit(origin_info, visit_ts=None, visit_id=None, snapshot_id=None): """Function that returns information about a visit for a given origin. The visit is retrieved from a provided timestamp. The closest visit from that timestamp is selected. Args: origin_info (dict): a dict filled with origin information visit_ts (int or str): an ISO date string or Unix timestamp to parse Returns: A dict containing the visit info as described below:: {'origin': 'https://forge.softwareheritage.org/source/swh-web/', 'date': '2017-10-08T11:54:25.582463+00:00', 'metadata': {}, 'visit': 25, 'status': 'full'} """ visits = get_origin_visits(origin_info) if not visits: - raise NotFoundExc(('No visit associated to origin with' - ' url %s!' % origin_info['url'])) + raise NotFoundExc( + ("No visit associated to origin with" " url %s!" % origin_info["url"]) + ) if snapshot_id: - visit = [v for v in visits if v['snapshot'] == snapshot_id] + visit = [v for v in visits if v["snapshot"] == snapshot_id] if len(visit) == 0: - raise NotFoundExc(('Visit for snapshot with id %s for origin with' - ' url %s not found!' % - (snapshot_id, origin_info['url']))) + raise NotFoundExc( + ( + "Visit for snapshot with id %s for origin with" + " url %s not found!" % (snapshot_id, origin_info["url"]) + ) + ) return visit[0] if visit_id: - visit = [v for v in visits if v['visit'] == int(visit_id)] + visit = [v for v in visits if v["visit"] == int(visit_id)] if len(visit) == 0: - raise NotFoundExc(('Visit with id %s for origin with' - ' url %s not found!' % - (visit_id, origin_info['url']))) + raise NotFoundExc( + ( + "Visit with id %s for origin with" + " url %s not found!" % (visit_id, origin_info["url"]) + ) + ) return visit[0] if not visit_ts: # returns the latest full visit when no timestamp is provided for v in reversed(visits): - if v['status'] == 'full': + if v["status"] == "full": return v return visits[-1] target_visit_ts = math.floor(parse_timestamp(visit_ts).timestamp()) # Find the visit with date closest to the target (in absolute value) (abs_time_delta, visit_idx) = min( - ((math.floor(parse_timestamp(visit['date']).timestamp()), i) - for (i, visit) in enumerate(visits)), - key=lambda ts_and_i: abs(ts_and_i[0] - target_visit_ts)) + ( + (math.floor(parse_timestamp(visit["date"]).timestamp()), i) + for (i, visit) in enumerate(visits) + ), + key=lambda ts_and_i: abs(ts_and_i[0] - target_visit_ts), + ) if visit_idx is not None: visit = visits[visit_idx] # If multiple visits have the same date, select the one with # the largest id. - while visit_idx < len(visits) - 1 and \ - visit['date'] == visits[visit_idx+1]['date']: + while ( + visit_idx < len(visits) - 1 + and visit["date"] == visits[visit_idx + 1]["date"] + ): visit_idx = visit_idx + 1 visit = visits[visit_idx] return visit else: - raise NotFoundExc(('Visit with timestamp %s for origin with ' - 'url %s not found!' % - (visit_ts, origin_info['url']))) + raise NotFoundExc( + ( + "Visit with timestamp %s for origin with " + "url %s not found!" % (visit_ts, origin_info["url"]) + ) + ) diff --git a/swh/web/common/query.py b/swh/web/common/query.py index 0e686c08..68b8c222 100644 --- a/swh/web/common/query.py +++ b/swh/web/common/query.py @@ -1,111 +1,113 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re from uuid import UUID from swh.model.hashutil import ALGORITHMS, hash_to_bytes from swh.web.common.exc import BadInputExc -SHA256_RE = re.compile(r'^[0-9a-f]{64}$', re.IGNORECASE) -SHA1_RE = re.compile(r'^[0-9a-f]{40}$', re.IGNORECASE) +SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) +SHA1_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) def parse_hash(q): """Detect the hash type of a user submitted query string. Args: query string with the following format: "[HASH_TYPE:]HEX_CHECKSUM", where HASH_TYPE is optional, defaults to "sha1", and can be one of swh.model.hashutil.ALGORITHMS Returns: A pair (hash_algorithm, byte hash value) Raises: ValueError if the given query string does not correspond to a valid hash value """ + def guess_algo(q): if SHA1_RE.match(q): - return 'sha1' + return "sha1" elif SHA256_RE.match(q): - return 'sha256' + return "sha256" else: - raise BadInputExc('Invalid checksum query string %s' % q) + raise BadInputExc("Invalid checksum query string %s" % q) def check_algo(algo, hex): - if (algo in {'sha1', 'sha1_git'} and not SHA1_RE.match(hex)) \ - or (algo == 'sha256' and not SHA256_RE.match(hex)): - raise BadInputExc('Invalid hash %s for algorithm %s' % (hex, algo)) + if (algo in {"sha1", "sha1_git"} and not SHA1_RE.match(hex)) or ( + algo == "sha256" and not SHA256_RE.match(hex) + ): + raise BadInputExc("Invalid hash %s for algorithm %s" % (hex, algo)) - parts = q.split(':') + parts = q.split(":") if len(parts) > 2: - raise BadInputExc('Invalid checksum query string %s' % q) + raise BadInputExc("Invalid checksum query string %s" % q) elif len(parts) == 1: parts = (guess_algo(q), q) elif len(parts) == 2: check_algo(parts[0], parts[1]) algo = parts[0] if algo not in ALGORITHMS: - raise BadInputExc('Unknown hash algorithm %s' % algo) + raise BadInputExc("Unknown hash algorithm %s" % algo) return (algo, hash_to_bytes(parts[1])) def parse_hash_with_algorithms_or_throws(q, accepted_algo, error_msg): """Parse a query but only accepts accepted_algo. Otherwise, raise the exception with message error_msg. Args: - q: query string with the following format: "[HASH_TYPE:]HEX_CHECKSUM" where HASH_TYPE is optional, defaults to "sha1", and can be one of swh.model.hashutil.ALGORITHMS. - accepted_algo: array of strings representing the names of accepted algorithms. - error_msg: error message to raise as BadInputExc if the algo of the query does not match. Returns: A pair (hash_algorithm, byte hash value) Raises: BadInputExc when the inputs is invalid or does not validate the accepted algorithms. """ algo, hash = parse_hash(q) if algo not in accepted_algo: raise BadInputExc(error_msg) return (algo, hash) def parse_uuid4(uuid): """Parse an uuid 4 from a string. Args: uuid: String representing an uuid. Returns: The uuid as is if everything is ok. Raises: BadInputExc: if the uuid is invalid. """ try: UUID(uuid, version=4) except ValueError as e: # not a valid hex code for a UUID raise BadInputExc(str(e)) return uuid diff --git a/swh/web/common/service.py b/swh/web/common/service.py index 6acc2a6c..7c5b0a48 100644 --- a/swh/web/common/service.py +++ b/swh/web/common/service.py @@ -1,1222 +1,1228 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import itertools import os import re from collections import defaultdict from typing import Any, Dict, List, Set from swh.model import hashutil from swh.storage.algos import diff, revisions_walker -from swh.model.identifiers import ( - CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT -) +from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT from swh.web.common import converters from swh.web.common import query from swh.web.common.exc import BadInputExc, NotFoundExc from swh.web.common.origin_visits import get_origin_visit from swh.web import config search = config.search() storage = config.storage() vault = config.vault() idx_storage = config.indexer_storage() MAX_LIMIT = 50 # Top limit the users can ask for def _first_element(l): """Returns the first element in the provided list or None if it is empty or None""" return next(iter(l or []), None) def lookup_multiple_hashes(hashes): """Lookup the passed hashes in a single DB connection, using batch processing. Args: An array of {filename: X, sha1: Y}, string X, hex sha1 string Y. Returns: The same array with elements updated with elem['found'] = true if the hash is present in storage, elem['found'] = false if not. """ - hashlist = [hashutil.hash_to_bytes(elem['sha1']) for elem in hashes] + hashlist = [hashutil.hash_to_bytes(elem["sha1"]) for elem in hashes] content_missing = storage.content_missing_per_sha1(hashlist) missing = [hashutil.hash_to_hex(x) for x in content_missing] for x in hashes: - x.update({'found': True}) + x.update({"found": True}) for h in hashes: - if h['sha1'] in missing: - h['found'] = False + if h["sha1"] in missing: + h["found"] = False return hashes def lookup_expression(expression, last_sha1, per_page): """Lookup expression in raw content. Args: expression (str): An expression to lookup through raw indexed content last_sha1 (str): Last sha1 seen per_page (int): Number of results per page Yields: ctags whose content match the expression """ limit = min(per_page, MAX_LIMIT) - ctags = idx_storage.content_ctags_search(expression, - last_sha1=last_sha1, - limit=limit) + ctags = idx_storage.content_ctags_search( + expression, last_sha1=last_sha1, limit=limit + ) for ctag in ctags: - ctag = converters.from_swh(ctag, hashess={'id'}) - ctag['sha1'] = ctag['id'] - ctag.pop('id') + ctag = converters.from_swh(ctag, hashess={"id"}) + ctag["sha1"] = ctag["id"] + ctag.pop("id") yield ctag def lookup_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found containing the hash info if the hash is present, None if not. """ algo, hash = query.parse_hash(q) found = _first_element(storage.content_find({algo: hash})) - return {'found': converters.from_content(found), - 'algo': algo} + return {"found": converters.from_content(found), "algo": algo} def search_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found to True or False, according to whether the checksum is present or not """ algo, hash = query.parse_hash(q) found = _first_element(storage.content_find({algo: hash})) - return {'found': found is not None} + return {"found": found is not None} def _lookup_content_sha1(q): """Given a possible input, query for the content's sha1. Args: q: query string of the form Returns: binary sha1 if found or None """ algo, hash = query.parse_hash(q) - if algo != 'sha1': + if algo != "sha1": hashes = _first_element(storage.content_find({algo: hash})) if not hashes: return None - return hashes['sha1'] + return hashes["sha1"] return hash def lookup_content_ctags(q): """Return ctags information from a specified content. Args: q: query string of the form Yields: ctags information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None ctags = list(idx_storage.content_ctags_get([sha1])) if not ctags: return None for ctag in ctags: - yield converters.from_swh(ctag, hashess={'id'}) + yield converters.from_swh(ctag, hashess={"id"}) def lookup_content_filetype(q): """Return filetype information from a specified content. Args: q: query string of the form Yields: filetype information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None filetype = _first_element(list(idx_storage.content_mimetype_get([sha1]))) if not filetype: return None return converters.from_filetype(filetype) def lookup_content_language(q): """Return language information from a specified content. Args: q: query string of the form Yields: language information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None lang = _first_element(list(idx_storage.content_language_get([sha1]))) if not lang: return None - return converters.from_swh(lang, hashess={'id'}) + return converters.from_swh(lang, hashess={"id"}) def lookup_content_license(q): """Return license information from a specified content. Args: q: query string of the form Yields: license information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None lic = _first_element(idx_storage.content_fossology_license_get([sha1])) if not lic: return None - return converters.from_swh({'id': sha1, 'facts': lic[sha1]}, - hashess={'id'}) + return converters.from_swh({"id": sha1, "facts": lic[sha1]}, hashess={"id"}) def lookup_origin(origin: Dict[str, str]) -> Dict[str, str]: """Return information about the origin matching dict origin. Args: origin: origin's dict with 'url' key Returns: origin information as dict. """ origins = [origin] - if origin['url']: + if origin["url"]: # handle case when user provided an origin url with a trailing # slash while the url in storage does not have it (e.g. GitHub) - if origin['url'].endswith('/'): - origins.append({'url': origin['url'][:-1]}) + if origin["url"].endswith("/"): + origins.append({"url": origin["url"][:-1]}) # handle case when user provided an origin url without a trailing # slash while the url in storage have it (e.g. Debian source package) else: - origins.append({'url': f"{origin['url']}/"}) + origins.append({"url": f"{origin['url']}/"}) # Check all possible origin urls for orig in origins: origin_info = storage.origin_get(orig) if origin_info: break if not origin_info: - msg = 'Origin with url %s not found!' % origin['url'] + msg = "Origin with url %s not found!" % origin["url"] raise NotFoundExc(msg) return converters.from_origin(origin_info) def lookup_origins(origin_from=1, origin_count=100): """Get list of archived software origins in a paginated way. Origins are sorted by id before returning them Args: origin_from (int): The minimum id of the origins to return origin_count (int): The maximum number of origins to return Yields: origins information as dicts """ origins = storage.origin_get_range(origin_from, origin_count) return map(converters.from_origin, origins) def search_origin(url_pattern, limit=50, with_visit=False, page_token=None): """Search for origins whose urls contain a provided string pattern or match a provided regular expression. Args: url_pattern: the string pattern to search for in origin urls limit: the maximum number of found origins to return page_token: opaque string used to get the next results of a search Returns: list of origin information as dict. """ if search: - results = search.origin_search(url_pattern=url_pattern, count=limit, - page_token=page_token, - with_visit=with_visit) - origins = list(map(converters.from_origin, results['results'])) - return (origins, results['next_page_token']) + results = search.origin_search( + url_pattern=url_pattern, + count=limit, + page_token=page_token, + with_visit=with_visit, + ) + origins = list(map(converters.from_origin, results["results"])) + return (origins, results["next_page_token"]) else: # Fallback to swh-storage if swh-search is not configured offset = int(page_token) if page_token else 0 regexp = True search_words = [re.escape(word) for word in url_pattern.split()] if len(search_words) >= 7: - url_pattern = '.*'.join(search_words) + url_pattern = ".*".join(search_words) else: pattern_parts = [] for permut in itertools.permutations(search_words): - pattern_parts.append('.*'.join(permut)) - url_pattern = '|'.join(pattern_parts) + pattern_parts.append(".*".join(permut)) + url_pattern = "|".join(pattern_parts) - origins = storage.origin_search(url_pattern, offset, limit, regexp, - with_visit) + origins = storage.origin_search(url_pattern, offset, limit, regexp, with_visit) origins = list(map(converters.from_origin, origins)) if len(origins) >= limit: page_token = str(offset + len(origins)) else: page_token = None return (origins, page_token) def search_origin_metadata(fulltext, limit=50): """Search for origins whose metadata match a provided string pattern. Args: fulltext: the string pattern to search for in origin metadata offset: number of found origins to skip before returning results limit: the maximum number of found origins to return Returns: list of origin metadata as dict. """ matches = idx_storage.origin_intrinsic_metadata_search_fulltext( - conjunction=[fulltext], limit=limit) + conjunction=[fulltext], limit=limit + ) results = [] for match in matches: - match['from_revision'] = hashutil.hash_to_hex(match['from_revision']) + match["from_revision"] = hashutil.hash_to_hex(match["from_revision"]) - origin = storage.origin_get({'url': match['id']}) - del match['id'] + origin = storage.origin_get({"url": match["id"]}) + del match["id"] result = converters.from_origin(origin) if result: - result['metadata'] = match + result["metadata"] = match results.append(result) return results def lookup_origin_intrinsic_metadata(origin_dict): """Return intrinsic metadata for origin whose origin matches given origin. Args: origin_dict: origin's dict with keys ('type' AND 'url') Returns: origin metadata. """ origin_info = storage.origin_get(origin_dict) if not origin_info: - msg = 'Origin with url %s not found!' % origin_dict['url'] + msg = "Origin with url %s not found!" % origin_dict["url"] raise NotFoundExc(msg) - origins = [origin_info['url']] - match = _first_element( - idx_storage.origin_intrinsic_metadata_get(origins)) + origins = [origin_info["url"]] + match = _first_element(idx_storage.origin_intrinsic_metadata_get(origins)) result = {} if match: - result = match['metadata'] + result = match["metadata"] return result def _to_sha1_bin(sha1_hex): _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( - sha1_hex, - ['sha1'], # HACK: sha1_git really - 'Only sha1_git is supported.') + sha1_hex, ["sha1"], "Only sha1_git is supported." # HACK: sha1_git really + ) return sha1_git_bin def _check_directory_exists(sha1_git, sha1_git_bin): if len(list(storage.directory_missing([sha1_git_bin]))): - raise NotFoundExc('Directory with sha1_git %s not found' % sha1_git) + raise NotFoundExc("Directory with sha1_git %s not found" % sha1_git) def lookup_directory(sha1_git): """Return information about the directory with id sha1_git. Args: sha1_git as string Returns: directory information as dict. """ - empty_dir_sha1 = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' + empty_dir_sha1 = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" if sha1_git == empty_dir_sha1: return [] sha1_git_bin = _to_sha1_bin(sha1_git) _check_directory_exists(sha1_git, sha1_git_bin) directory_entries = storage.directory_ls(sha1_git_bin) return map(converters.from_directory_entry, directory_entries) def lookup_directory_with_path(sha1_git, path_string): """Return directory information for entry with path path_string w.r.t. root directory pointed by directory_sha1_git Args: - directory_sha1_git: sha1_git corresponding to the directory to which we append paths to (hopefully) find the entry - the relative path to the entry starting from the directory pointed by directory_sha1_git Raises: NotFoundExc if the directory entry is not found """ sha1_git_bin = _to_sha1_bin(sha1_git) _check_directory_exists(sha1_git, sha1_git_bin) paths = path_string.strip(os.path.sep).split(os.path.sep) queried_dir = storage.directory_entry_get_by_path( - sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) + sha1_git_bin, list(map(lambda p: p.encode("utf-8"), paths)) + ) if not queried_dir: - raise NotFoundExc(('Directory entry with path %s from %s not found') % - (path_string, sha1_git)) + raise NotFoundExc( + ("Directory entry with path %s from %s not found") % (path_string, sha1_git) + ) return converters.from_directory_entry(queried_dir) def lookup_release(release_sha1_git): """Return information about the release with sha1 release_sha1_git. Args: release_sha1_git: The release's sha1 as hexadecimal Returns: Release information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_git_bin = _to_sha1_bin(release_sha1_git) release = _first_element(storage.release_get([sha1_git_bin])) if not release: - raise NotFoundExc('Release with sha1_git %s not found.' - % release_sha1_git) + raise NotFoundExc("Release with sha1_git %s not found." % release_sha1_git) return converters.from_release(release) def lookup_release_multiple(sha1_git_list): """Return information about the revisions identified with their sha1_git identifiers. Args: sha1_git_list: A list of revision sha1_git identifiers Returns: Release information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_bin_list = (_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list) releases = storage.release_get(sha1_bin_list) or [] return (converters.from_release(r) for r in releases) def lookup_revision(rev_sha1_git): """Return information about the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Revision information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. NotFoundExc if there is no revision with the provided sha1_git. """ sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: - raise NotFoundExc('Revision with sha1_git %s not found.' - % rev_sha1_git) + raise NotFoundExc("Revision with sha1_git %s not found." % rev_sha1_git) return converters.from_revision(revision) def lookup_revision_multiple(sha1_git_list): """Return information about the revisions identified with their sha1_git identifiers. Args: sha1_git_list: A list of revision sha1_git identifiers Returns: Generator of revisions information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ sha1_bin_list = (_to_sha1_bin(sha1_git) for sha1_git in sha1_git_list) revisions = storage.revision_get(sha1_bin_list) or [] return (converters.from_revision(r) for r in revisions) def lookup_revision_message(rev_sha1_git): """Return the raw message of the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Decoded revision message as dict {'message': } Raises: ValueError if the identifier provided is not of sha1 nature. NotFoundExc if the revision is not found, or if it has no message """ sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: - raise NotFoundExc('Revision with sha1_git %s not found.' - % rev_sha1_git) - if 'message' not in revision: - raise NotFoundExc('No message for revision with sha1_git %s.' - % rev_sha1_git) - res = {'message': revision['message']} + raise NotFoundExc("Revision with sha1_git %s not found." % rev_sha1_git) + if "message" not in revision: + raise NotFoundExc("No message for revision with sha1_git %s." % rev_sha1_git) + res = {"message": revision["message"]} return res def _lookup_revision_id_by(origin, branch_name, timestamp): def _get_snapshot_branch(snapshot, branch_name): - snapshot = lookup_snapshot(visit['snapshot'], - branches_from=branch_name, - branches_count=10) + snapshot = lookup_snapshot( + visit["snapshot"], branches_from=branch_name, branches_count=10 + ) branch = None - if branch_name in snapshot['branches']: - branch = snapshot['branches'][branch_name] + if branch_name in snapshot["branches"]: + branch = snapshot["branches"][branch_name] return branch if isinstance(origin, int): - origin = {'id': origin} + origin = {"id": origin} elif isinstance(origin, str): - origin = {'url': origin} + origin = {"url": origin} else: raise TypeError('"origin" must be an int or a string.') visit = get_origin_visit(origin, visit_ts=timestamp) - branch = _get_snapshot_branch(visit['snapshot'], branch_name) + branch = _get_snapshot_branch(visit["snapshot"], branch_name) rev_id = None - if branch and branch['target_type'] == 'revision': - rev_id = branch['target'] - elif branch and branch['target_type'] == 'alias': - branch = _get_snapshot_branch(visit['snapshot'], branch['target']) - if branch and branch['target_type'] == 'revision': - rev_id = branch['target'] + if branch and branch["target_type"] == "revision": + rev_id = branch["target"] + elif branch and branch["target_type"] == "alias": + branch = _get_snapshot_branch(visit["snapshot"], branch["target"]) + if branch and branch["target_type"] == "revision": + rev_id = branch["target"] if not rev_id: - raise NotFoundExc('Revision for origin %s and branch %s not found.' - % (origin.get('url'), branch_name)) + raise NotFoundExc( + "Revision for origin %s and branch %s not found." + % (origin.get("url"), branch_name) + ) return rev_id -def lookup_revision_by(origin, - branch_name='HEAD', - timestamp=None): +def lookup_revision_by(origin, branch_name="HEAD", timestamp=None): """Lookup revision by origin, snapshot branch name and visit timestamp. If branch_name is not provided, lookup using 'HEAD' as default. If timestamp is not provided, use the most recent. Args: origin (Union[int,str]): origin of the revision branch_name (str): snapshot branch name timestamp (str/int): origin visit time frame Returns: dict: The revision matching the criterions Raises: NotFoundExc if no revision corresponds to the criterion """ rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) return lookup_revision(rev_id) def lookup_revision_log(rev_sha1_git, limit): """Lookup revision log by revision id. Args: rev_sha1_git (str): The revision's sha1 as hexadecimal limit (int): the maximum number of revisions returned Returns: list: Revision log as list of revision dicts Raises: ValueError: if the identifier provided is not of sha1 nature. swh.web.common.exc.NotFoundExc: if there is no revision with the provided sha1_git. """ lookup_revision(rev_sha1_git) sha1_git_bin = _to_sha1_bin(rev_sha1_git) revision_entries = storage.revision_log([sha1_git_bin], limit) return map(converters.from_revision, revision_entries) def lookup_revision_log_by(origin, branch_name, timestamp, limit): """Lookup revision by origin, snapshot branch name and visit timestamp. Args: origin (Union[int,str]): origin of the revision branch_name (str): snapshot branch timestamp (str/int): origin visit time frame limit (int): the maximum number of revisions returned Returns: list: Revision log as list of revision dicts Raises: swh.web.common.exc.NotFoundExc: if no revision corresponds to the criterion """ rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) return lookup_revision_log(rev_id, limit) -def lookup_revision_with_context_by(origin, branch_name, timestamp, - sha1_git, limit=100): +def lookup_revision_with_context_by( + origin, branch_name, timestamp, sha1_git, limit=100 +): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. sha1_git_root being resolved through the lookup of a revision by origin, branch_name and ts. In other words, sha1_git is an ancestor of sha1_git_root. Args: - origin: origin of the revision. - branch_name: revision's branch. - timestamp: revision's time frame. - sha1_git: one of sha1_git_root's ancestors. - limit: limit the lookup to 100 revisions back. Returns: Pair of (root_revision, revision). Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: - BadInputExc in case of unknown algo_hash or bad hash. - NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root. """ rev_root_id = _lookup_revision_id_by(origin, branch_name, timestamp) rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) rev_root = _first_element(storage.revision_get([rev_root_id_bin])) - return (converters.from_revision(rev_root), - lookup_revision_with_context(rev_root, sha1_git, limit)) + return ( + converters.from_revision(rev_root), + lookup_revision_with_context(rev_root, sha1_git, limit), + ) def lookup_revision_with_context(sha1_git_root, sha1_git, limit=100): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. In other words, sha1_git is an ancestor of sha1_git_root. Args: sha1_git_root: latest revision. The type is either a sha1 (as an hex string) or a non converted dict. sha1_git: one of sha1_git_root's ancestors limit: limit the lookup to 100 revisions back Returns: Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root """ sha1_git_bin = _to_sha1_bin(sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: - raise NotFoundExc('Revision %s not found' % sha1_git) + raise NotFoundExc("Revision %s not found" % sha1_git) if isinstance(sha1_git_root, str): sha1_git_root_bin = _to_sha1_bin(sha1_git_root) - revision_root = _first_element(storage.revision_get([sha1_git_root_bin])) # noqa + revision_root = _first_element(storage.revision_get([sha1_git_root_bin])) if not revision_root: - raise NotFoundExc('Revision root %s not found' % sha1_git_root) + raise NotFoundExc("Revision root %s not found" % sha1_git_root) else: - sha1_git_root_bin = sha1_git_root['id'] + sha1_git_root_bin = sha1_git_root["id"] revision_log = storage.revision_log([sha1_git_root_bin], limit) parents = {} children = defaultdict(list) for rev in revision_log: - rev_id = rev['id'] + rev_id = rev["id"] parents[rev_id] = [] - for parent_id in rev['parents']: + for parent_id in rev["parents"]: parents[rev_id].append(parent_id) children[parent_id].append(rev_id) - if revision['id'] not in parents: - raise NotFoundExc('Revision %s is not an ancestor of %s' % - (sha1_git, sha1_git_root)) + if revision["id"] not in parents: + raise NotFoundExc( + "Revision %s is not an ancestor of %s" % (sha1_git, sha1_git_root) + ) - revision['children'] = children[revision['id']] + revision["children"] = children[revision["id"]] return converters.from_revision(revision) def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False): """Return information on directory pointed by revision with sha1_git. If dir_path is not provided, display top level directory. Otherwise, display the directory pointed by dir_path (if it exists). Args: sha1_git: revision's hash. dir_path: optional directory pointed to by that revision. with_data: boolean that indicates to retrieve the raw data if the path resolves to a content. Default to False (for the api) Returns: Information on the directory pointed to by that revision. Raises: BadInputExc in case of unknown algo_hash or bad hash. NotFoundExc either if the revision is not found or the path referenced does not exist. NotImplementedError in case of dir_path exists but do not reference a type 'dir' or 'file'. """ sha1_git_bin = _to_sha1_bin(sha1_git) revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: - raise NotFoundExc('Revision %s not found' % sha1_git) - dir_sha1_git_bin = revision['directory'] + raise NotFoundExc("Revision %s not found" % sha1_git) + dir_sha1_git_bin = revision["directory"] if dir_path: paths = dir_path.strip(os.path.sep).split(os.path.sep) entity = storage.directory_entry_get_by_path( - dir_sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) + dir_sha1_git_bin, list(map(lambda p: p.encode("utf-8"), paths)) + ) if not entity: raise NotFoundExc( "Directory or File '%s' pointed to by revision %s not found" - % (dir_path, sha1_git)) + % (dir_path, sha1_git) + ) else: - entity = {'type': 'dir', 'target': dir_sha1_git_bin} - if entity['type'] == 'dir': - directory_entries = storage.directory_ls(entity['target']) or [] - return {'type': 'dir', - 'path': '.' if not dir_path else dir_path, - 'revision': sha1_git, - 'content': list(map(converters.from_directory_entry, - directory_entries))} - elif entity['type'] == 'file': # content - content = _first_element( - storage.content_find({'sha1_git': entity['target']})) + entity = {"type": "dir", "target": dir_sha1_git_bin} + if entity["type"] == "dir": + directory_entries = storage.directory_ls(entity["target"]) or [] + return { + "type": "dir", + "path": "." if not dir_path else dir_path, + "revision": sha1_git, + "content": list(map(converters.from_directory_entry, directory_entries)), + } + elif entity["type"] == "file": # content + content = _first_element(storage.content_find({"sha1_git": entity["target"]})) if not content: - raise NotFoundExc('Content not found for revision %s' - % sha1_git) + raise NotFoundExc("Content not found for revision %s" % sha1_git) if with_data: - c = _first_element(storage.content_get([content['sha1']])) - content['data'] = c['data'] - return {'type': 'file', - 'path': '.' if not dir_path else dir_path, - 'revision': sha1_git, - 'content': converters.from_content(content)} - elif entity['type'] == 'rev': # revision - revision = next(storage.revision_get([entity['target']])) - return {'type': 'rev', - 'path': '.' if not dir_path else dir_path, - 'revision': sha1_git, - 'content': converters.from_revision(revision)} + c = _first_element(storage.content_get([content["sha1"]])) + content["data"] = c["data"] + return { + "type": "file", + "path": "." if not dir_path else dir_path, + "revision": sha1_git, + "content": converters.from_content(content), + } + elif entity["type"] == "rev": # revision + revision = next(storage.revision_get([entity["target"]])) + return { + "type": "rev", + "path": "." if not dir_path else dir_path, + "revision": sha1_git, + "content": converters.from_revision(revision), + } else: - raise NotImplementedError('Entity of type %s not implemented.' - % entity['type']) + raise NotImplementedError("Entity of type %s not implemented." % entity["type"]) def lookup_content(q): """Lookup the content designed by q. Args: q: The release's sha1 as hexadecimal Raises: NotFoundExc if the requested content is not found """ algo, hash = query.parse_hash(q) c = _first_element(storage.content_find({algo: hash})) if not c: - raise NotFoundExc('Content with %s checksum equals to %s not found!' % - (algo, hashutil.hash_to_hex(hash))) + raise NotFoundExc( + "Content with %s checksum equals to %s not found!" + % (algo, hashutil.hash_to_hex(hash)) + ) return converters.from_content(c) def lookup_content_raw(q): """Lookup the content defined by q. Args: q: query string of the form Returns: dict with 'sha1' and 'data' keys. data representing its raw data decoded. Raises: NotFoundExc if the requested content is not found or if the content bytes are not available in the storage """ c = lookup_content(q) - content_sha1_bytes = hashutil.hash_to_bytes(c['checksums']['sha1']) + content_sha1_bytes = hashutil.hash_to_bytes(c["checksums"]["sha1"]) content = _first_element(storage.content_get([content_sha1_bytes])) if not content: algo, hash = query.parse_hash(q) - raise NotFoundExc('Bytes of content with %s checksum equals to %s ' - 'are not available!' % - (algo, hashutil.hash_to_hex(hash))) + raise NotFoundExc( + "Bytes of content with %s checksum equals to %s " + "are not available!" % (algo, hashutil.hash_to_hex(hash)) + ) return converters.from_content(content) def stat_counters(): """Return the stat counters for Software Heritage Returns: A dict mapping textual labels to integer values. """ return storage.stat_counters() def _lookup_origin_visits(origin_url, last_visit=None, limit=10): """Yields the origin origins' visits. Args: origin_url (str): origin to list visits for last_visit (int): last visit to lookup from limit (int): Number of elements max to display Yields: Dictionaries of origin_visit for that origin """ limit = min(limit, MAX_LIMIT) for visit in storage.origin_visit_get( - origin_url, last_visit=last_visit, limit=limit): - visit['origin'] = origin_url + origin_url, last_visit=last_visit, limit=limit + ): + visit["origin"] = origin_url yield visit def lookup_origin_visits(origin, last_visit=None, per_page=10): """Yields the origin origins' visits. Args: origin: origin to list visits for Yields: Dictionaries of origin_visit for that origin """ - visits = _lookup_origin_visits(origin, last_visit=last_visit, - limit=per_page) + visits = _lookup_origin_visits(origin, last_visit=last_visit, limit=per_page) for visit in visits: yield converters.from_origin_visit(visit) def lookup_origin_visit_latest(origin_url, require_snapshot): """Return the origin's latest visit Args: origin_url (str): origin to list visits for require_snapshot (bool): filter out origins without a snapshot Returns: dict: The origin_visit concerned """ visit = storage.origin_visit_get_latest( - origin_url, require_snapshot=require_snapshot) + origin_url, require_snapshot=require_snapshot + ) return converters.from_origin_visit(visit) def lookup_origin_visit(origin_url, visit_id): """Return information about visit visit_id with origin origin. Args: origin (str): origin concerned by the visit visit_id: the visit identifier to lookup Yields: The dict origin_visit concerned """ visit = storage.origin_visit_get_by(origin_url, visit_id) if not visit: - raise NotFoundExc('Origin %s or its visit ' - 'with id %s not found!' % (origin_url, visit_id)) - visit['origin'] = origin_url + raise NotFoundExc( + "Origin %s or its visit " "with id %s not found!" % (origin_url, visit_id) + ) + visit["origin"] = origin_url return converters.from_origin_visit(visit) def lookup_snapshot_sizes(snapshot_id): """Count the number of branches in the snapshot with the given id Args: snapshot_id (str): sha1 identifier of the snapshot Returns: dict: A dict whose keys are the target types of branches and values their corresponding amount """ snapshot_id_bin = _to_sha1_bin(snapshot_id) snapshot_sizes = storage.snapshot_count_branches(snapshot_id_bin) - if 'revision' not in snapshot_sizes: - snapshot_sizes['revision'] = 0 - if 'release' not in snapshot_sizes: - snapshot_sizes['release'] = 0 + if "revision" not in snapshot_sizes: + snapshot_sizes["revision"] = 0 + if "release" not in snapshot_sizes: + snapshot_sizes["release"] = 0 # adjust revision / release count for display if aliases are defined - if 'alias' in snapshot_sizes: - aliases = lookup_snapshot(snapshot_id, - branches_count=snapshot_sizes['alias'], - target_types=['alias']) - for alias in aliases['branches'].values(): - if lookup_snapshot(snapshot_id, - branches_from=alias['target'], - branches_count=1, - target_types=['revision']): - snapshot_sizes['revision'] += 1 + if "alias" in snapshot_sizes: + aliases = lookup_snapshot( + snapshot_id, branches_count=snapshot_sizes["alias"], target_types=["alias"] + ) + for alias in aliases["branches"].values(): + if lookup_snapshot( + snapshot_id, + branches_from=alias["target"], + branches_count=1, + target_types=["revision"], + ): + snapshot_sizes["revision"] += 1 else: - snapshot_sizes['release'] += 1 - del snapshot_sizes['alias'] + snapshot_sizes["release"] += 1 + del snapshot_sizes["alias"] return snapshot_sizes -def lookup_snapshot(snapshot_id, branches_from='', branches_count=1000, - target_types=None): +def lookup_snapshot( + snapshot_id, branches_from="", branches_count=1000, target_types=None +): """Return information about a snapshot, aka the list of named branches found during a specific visit of an origin. Args: snapshot_id (str): sha1 identifier of the snapshot branches_from (str): optional parameter used to skip branches whose name is lesser than it before returning them branches_count (int): optional parameter used to restrain the amount of returned branches target_types (list): optional parameter used to filter the target types of branch to return (possible values that can be contained in that list are `'content', 'directory', 'revision', 'release', 'snapshot', 'alias'`) Returns: A dict filled with the snapshot content. """ snapshot_id_bin = _to_sha1_bin(snapshot_id) - snapshot = storage.snapshot_get_branches(snapshot_id_bin, - branches_from.encode(), - branches_count, target_types) + snapshot = storage.snapshot_get_branches( + snapshot_id_bin, branches_from.encode(), branches_count, target_types + ) if not snapshot: - raise NotFoundExc('Snapshot with id %s not found!' % snapshot_id) + raise NotFoundExc("Snapshot with id %s not found!" % snapshot_id) return converters.from_snapshot(snapshot) def lookup_latest_origin_snapshot(origin, allowed_statuses=None): """Return information about the latest snapshot of an origin. .. warning:: At most 1000 branches contained in the snapshot will be returned for performance reasons. Args: origin: URL or integer identifier of the origin allowed_statuses: list of visit statuses considered to find the latest snapshot for the visit. For instance, ``allowed_statuses=['full']`` will only consider visits that have successfully run to completion. Returns: A dict filled with the snapshot content. """ snapshot = storage.snapshot_get_latest(origin, allowed_statuses) return converters.from_snapshot(snapshot) def lookup_revision_through(revision, limit=100): """Retrieve a revision from the criterion stored in revision dictionary. Args: revision: Dictionary of criterion to lookup the revision with. Here are the supported combination of possible values: - origin_url, branch_name, ts, sha1_git - origin_url, branch_name, ts - sha1_git_root, sha1_git - sha1_git Returns: None if the revision is not found or the actual revision. """ if ( - 'origin_url' in revision and - 'branch_name' in revision and - 'ts' in revision and - 'sha1_git' in revision): - return lookup_revision_with_context_by(revision['origin_url'], - revision['branch_name'], - revision['ts'], - revision['sha1_git'], - limit) - if ( - 'origin_url' in revision and - 'branch_name' in revision and - 'ts' in revision): - return lookup_revision_by(revision['origin_url'], - revision['branch_name'], - revision['ts']) - if ( - 'sha1_git_root' in revision and - 'sha1_git' in revision): - return lookup_revision_with_context(revision['sha1_git_root'], - revision['sha1_git'], - limit) - if 'sha1_git' in revision: - return lookup_revision(revision['sha1_git']) + "origin_url" in revision + and "branch_name" in revision + and "ts" in revision + and "sha1_git" in revision + ): + return lookup_revision_with_context_by( + revision["origin_url"], + revision["branch_name"], + revision["ts"], + revision["sha1_git"], + limit, + ) + if "origin_url" in revision and "branch_name" in revision and "ts" in revision: + return lookup_revision_by( + revision["origin_url"], revision["branch_name"], revision["ts"] + ) + if "sha1_git_root" in revision and "sha1_git" in revision: + return lookup_revision_with_context( + revision["sha1_git_root"], revision["sha1_git"], limit + ) + if "sha1_git" in revision: + return lookup_revision(revision["sha1_git"]) # this should not happen - raise NotImplementedError('Should not happen!') + raise NotImplementedError("Should not happen!") -def lookup_directory_through_revision(revision, path=None, - limit=100, with_data=False): +def lookup_directory_through_revision(revision, path=None, limit=100, with_data=False): """Retrieve the directory information from the revision. Args: revision: dictionary of criterion representing a revision to lookup path: directory's path to lookup. limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of. with_data: indicate to retrieve the content's raw data if path resolves to a content. Returns: The directory pointing to by the revision criterions at path. """ rev = lookup_revision_through(revision, limit) if not rev: - raise NotFoundExc('Revision with criterion %s not found!' % revision) - return (rev['id'], - lookup_directory_with_revision(rev['id'], path, with_data)) + raise NotFoundExc("Revision with criterion %s not found!" % revision) + return (rev["id"], lookup_directory_with_revision(rev["id"], path, with_data)) def vault_cook(obj_type, obj_id, email=None): """Cook a vault bundle. """ return vault.cook(obj_type, obj_id, email=email) def vault_fetch(obj_type, obj_id): """Fetch a vault bundle. """ return vault.fetch(obj_type, obj_id) def vault_progress(obj_type, obj_id): """Get the current progress of a vault bundle. """ return vault.progress(obj_type, obj_id) def diff_revision(rev_id): """Get the list of file changes (insertion / deletion / modification / renaming) for a particular revision. """ rev_sha1_git_bin = _to_sha1_bin(rev_id) - changes = diff.diff_revision(storage, rev_sha1_git_bin, - track_renaming=True) + changes = diff.diff_revision(storage, rev_sha1_git_bin, track_renaming=True) for change in changes: - change['from'] = converters.from_directory_entry(change['from']) - change['to'] = converters.from_directory_entry(change['to']) - if change['from_path']: - change['from_path'] = change['from_path'].decode('utf-8') - if change['to_path']: - change['to_path'] = change['to_path'].decode('utf-8') + change["from"] = converters.from_directory_entry(change["from"]) + change["to"] = converters.from_directory_entry(change["to"]) + if change["from_path"]: + change["from_path"] = change["from_path"].decode("utf-8") + if change["to_path"]: + change["to_path"] = change["to_path"].decode("utf-8") return changes class _RevisionsWalkerProxy(object): """ Proxy class wrapping a revisions walker iterator from swh-storage and performing needed conversions. """ + def __init__(self, rev_walker_type, rev_start, *args, **kwargs): rev_start_bin = hashutil.hash_to_bytes(rev_start) - self.revisions_walker = \ - revisions_walker.get_revisions_walker(rev_walker_type, - storage, - rev_start_bin, - *args, **kwargs) + self.revisions_walker = revisions_walker.get_revisions_walker( + rev_walker_type, storage, rev_start_bin, *args, **kwargs + ) def export_state(self): return self.revisions_walker.export_state() def __next__(self): return converters.from_revision(next(self.revisions_walker)) def __iter__(self): return self def get_revisions_walker(rev_walker_type, rev_start, *args, **kwargs): """ Utility function to instantiate a revisions walker of a given type, see :mod:`swh.storage.algos.revisions_walker`. Args: rev_walker_type (str): the type of revisions walker to return, possible values are: ``committer_date``, ``dfs``, ``dfs_post``, ``bfs`` and ``path`` rev_start (str): hexadecimal representation of a revision identifier args (list): position arguments to pass to the revisions walker constructor kwargs (dict): keyword arguments to pass to the revisions walker constructor """ # first check if the provided revision is valid lookup_revision(rev_start) return _RevisionsWalkerProxy(rev_walker_type, rev_start, *args, **kwargs) def lookup_object(object_type: str, object_id: str) -> Dict[str, Any]: """ Utility function for looking up an object in the archive by its type and id. Args: object_type (str): the type of object to lookup, either *content*, *directory*, *release*, *revision* or *snapshot* object_id (str): the *sha1_git* checksum identifier in hexadecimal form of the object to lookup Returns: Dict[str, Any]: A dictionary describing the object or a list of dictionary for the directory object type. Raises: swh.web.common.exc.NotFoundExc: if the object could not be found in the archive BadInputExc: if the object identifier is invalid """ if object_type == CONTENT: - return lookup_content(f'sha1_git:{object_id}') + return lookup_content(f"sha1_git:{object_id}") elif object_type == DIRECTORY: - return { - 'id': object_id, - 'content': list(lookup_directory(object_id)) - } + return {"id": object_id, "content": list(lookup_directory(object_id))} elif object_type == RELEASE: return lookup_release(object_id) elif object_type == REVISION: return lookup_revision(object_id) elif object_type == SNAPSHOT: return lookup_snapshot(object_id) - raise BadInputExc(('Invalid swh object type! Valid types are ' - f'{CONTENT}, {DIRECTORY}, {RELEASE} ' - f'{REVISION} or {SNAPSHOT}.')) + raise BadInputExc( + ( + "Invalid swh object type! Valid types are " + f"{CONTENT}, {DIRECTORY}, {RELEASE} " + f"{REVISION} or {SNAPSHOT}." + ) + ) def lookup_missing_hashes(grouped_pids: Dict[str, List[bytes]]) -> Set[str]: """Lookup missing Software Heritage persistent identifier hash, using batch processing. Args: A dictionary with: keys: persistent identifier type values: list(bytes) persistent identifier hash Returns: A set(hexadecimal) of the hashes not found in the storage """ missing_hashes = [] for obj_type, obj_ids in grouped_pids.items(): if obj_type == CONTENT: - missing_hashes.append( - storage.content_missing_per_sha1_git(obj_ids)) + missing_hashes.append(storage.content_missing_per_sha1_git(obj_ids)) elif obj_type == DIRECTORY: missing_hashes.append(storage.directory_missing(obj_ids)) elif obj_type == REVISION: missing_hashes.append(storage.revision_missing(obj_ids)) elif obj_type == RELEASE: missing_hashes.append(storage.directory_missing(obj_ids)) elif obj_type == SNAPSHOT: missing_hashes.append(storage.directory_missing(obj_ids)) - missing = set(map(lambda x: hashutil.hash_to_hex(x), - itertools.chain(*missing_hashes))) + missing = set( + map(lambda x: hashutil.hash_to_hex(x), itertools.chain(*missing_hashes)) + ) return missing diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py index b2e9c0c4..11fb90ea 100644 --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -1,158 +1,157 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import re from django import template from django.core.serializers.json import DjangoJSONEncoder from django.utils.safestring import mark_safe import sentry_sdk from swh.web.common.origin_save import get_savable_visit_types from swh.web.common.utils import rst_to_html register = template.Library() @register.filter def docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ return rst_to_html(docstring) @register.filter def urlize_links_and_mails(text): """Utility function for decorating api links in browsable api. Args: text: whose content matching links should be transformed into contextual API or Browse html links. Returns The text transformed if any link is found. The text as is otherwise. """ try: if 'href="' not in text: - text = re.sub(r'(http.*)', r'\1', text) - return re.sub(r'([^ <>"]+@[^ <>"]+)', - r'\1', text) + text = re.sub(r"(http.*)", r'\1', text) + return re.sub(r'([^ <>"]+@[^ <>"]+)', r'\1', text) except Exception as exc: sentry_sdk.capture_exception(exc) return text @register.filter def urlize_header_links(text): """Utility function for decorating headers links in browsable api. Args text: Text whose content contains Link header value Returns: The text transformed with html link if any link is found. The text as is otherwise. """ - links = text.split(',') - ret = '' + links = text.split(",") + ret = "" for i, link in enumerate(links): - ret += re.sub(r'<(http.*)>', r'<\1>', link) + ret += re.sub(r"<(http.*)>", r'<\1>', link) # add one link per line and align them if i != len(links) - 1: - ret += '\n ' + ret += "\n " return ret @register.filter def jsonify(obj): """Utility function for converting a django template variable to JSON in order to use it in script tags. Args obj: Any django template context variable Returns: JSON representation of the variable. """ return mark_safe(json.dumps(obj, cls=DjangoJSONEncoder)) @register.filter def sub(value, arg): """Django template filter for subtracting two numbers Args: value (int/float): the value to subtract from arg (int/float): the value to subtract to Returns: int/float: The subtraction result """ return value - arg @register.filter def mul(value, arg): """Django template filter for multiplying two numbers Args: value (int/float): the value to multiply from arg (int/float): the value to multiply with Returns: int/float: The multiplication result """ return value * arg @register.filter def key_value(dict, key): """Django template filter to get a value in a dictionary. Args: dict (dict): a dictionary key (str): the key to lookup value Returns: The requested value in the dictionary """ return dict[key] @register.filter def visit_type_savable(visit_type): """Django template filter to check if a save request can be created for a given visit type. Args: visit_type (str): the type of visit Returns: If the visit type is saveable or not """ return visit_type in get_savable_visit_types() @register.filter def split(value, arg): """Django template filter to split a string. Args: value (str): the string to split arg (str): the split separator Returns: list: the split string parts """ return value.split(arg) diff --git a/swh/web/common/urlsindex.py b/swh/web/common/urlsindex.py index 08000426..32a78b53 100644 --- a/swh/web/common/urlsindex.py +++ b/swh/web/common/urlsindex.py @@ -1,80 +1,76 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict, List import django.urls from django.conf.urls import url from django.shortcuts import redirect class UrlsIndex(object): """ Simple helper class for centralizing url patterns of a Django web application. Derived classes should override the 'scope' class attribute otherwise all declared patterns will be grouped under the default one. """ _urlpatterns = {} # type: Dict[str, List[django.urls.URLPattern]] - scope = 'default' + scope = "default" @classmethod def add_url_pattern(cls, url_pattern, view, view_name=None): """ Class method that adds an url pattern to the current scope. Args: url_pattern: regex describing a Django url view: function implementing the Django view view_name: name of the view used to reverse the url """ if cls.scope not in cls._urlpatterns: cls._urlpatterns[cls.scope] = [] if view_name: - cls._urlpatterns[cls.scope].append(url(url_pattern, view, - name=view_name)) + cls._urlpatterns[cls.scope].append(url(url_pattern, view, name=view_name)) else: cls._urlpatterns[cls.scope].append(url(url_pattern, view)) @classmethod - def add_redirect_for_checksum_args(cls, view_name, url_patterns, - checksum_args): + def add_redirect_for_checksum_args(cls, view_name, url_patterns, checksum_args): """ Class method that redirects to view with lowercase checksums when upper/mixed case checksums are passed as url arguments. Args: view_name (str): name of the view to redirect requests url_patterns (List[str]): regexps describing the view urls checksum_args (List[str]): url argument names corresponding to checksum values """ - new_view_name = view_name+'-uppercase-checksum' + new_view_name = view_name + "-uppercase-checksum" for url_pattern in url_patterns: - url_pattern_upper = url_pattern.replace('[0-9a-f]', - '[0-9a-fA-F]') + url_pattern_upper = url_pattern.replace("[0-9a-f]", "[0-9a-fA-F]") def view_redirect(request, *args, **kwargs): for checksum_arg in checksum_args: checksum_upper = kwargs[checksum_arg] kwargs[checksum_arg] = checksum_upper.lower() return redirect(view_name, *args, **kwargs) - cls.add_url_pattern(url_pattern_upper, view_redirect, - new_view_name) + cls.add_url_pattern(url_pattern_upper, view_redirect, new_view_name) @classmethod def get_url_patterns(cls): """ Class method that returns the list of url pattern associated to the current scope. Returns: The list of url patterns associated to the current scope """ return cls._urlpatterns[cls.scope] diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py index 561b6989..c4405613 100644 --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -1,342 +1,346 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re from datetime import datetime, timezone from dateutil import parser as date_parser from dateutil import tz from typing import Optional, Dict, Any import docutils.parsers.rst import docutils.utils from bs4 import BeautifulSoup from docutils.core import publish_parts from docutils.writers.html5_polyglot import Writer, HTMLTranslator from django.urls import reverse as django_reverse from django.http import QueryDict, HttpRequest from prometheus_client.registry import CollectorRegistry from rest_framework.authentication import SessionAuthentication from swh.web.common.exc import BadInputExc from swh.web.common.typing import QueryParameters from swh.web.config import get_config SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) swh_object_icons = { - 'branch': 'fa fa-code-fork', - 'branches': 'fa fa-code-fork', - 'content': 'fa fa-file-text', - 'directory': 'fa fa-folder', - 'person': 'fa fa-user', - 'revisions history': 'fa fa-history', - 'release': 'fa fa-tag', - 'releases': 'fa fa-tag', - 'revision': 'octicon-git-commit', - 'snapshot': 'fa fa-camera', - 'visits': 'fa fa-calendar', + "branch": "fa fa-code-fork", + "branches": "fa fa-code-fork", + "content": "fa fa-file-text", + "directory": "fa fa-folder", + "person": "fa fa-user", + "revisions history": "fa fa-history", + "release": "fa fa-tag", + "releases": "fa fa-tag", + "revision": "octicon-git-commit", + "snapshot": "fa fa-camera", + "visits": "fa fa-calendar", } -def reverse(viewname: str, - url_args: Optional[Dict[str, Any]] = None, - query_params: Optional[QueryParameters] = None, - current_app: Optional[str] = None, - urlconf: Optional[str] = None, - request: Optional[HttpRequest] = None) -> str: +def reverse( + viewname: str, + url_args: Optional[Dict[str, Any]] = None, + query_params: Optional[QueryParameters] = None, + current_app: Optional[str] = None, + urlconf: Optional[str] = None, + request: Optional[HttpRequest] = None, +) -> str: """An override of django reverse function supporting query parameters. Args: viewname: the name of the django view from which to compute a url url_args: dictionary of url arguments indexed by their names query_params: dictionary of query parameters to append to the reversed url current_app: the name of the django app tighten to the view urlconf: url configuration module request: build an absolute URI if provided Returns: str: the url of the requested view with processed arguments and query parameters """ if url_args: url_args = {k: v for k, v in url_args.items() if v is not None} - url = django_reverse(viewname, urlconf=urlconf, kwargs=url_args, - current_app=current_app) + url = django_reverse( + viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app + ) if query_params: query_params = {k: v for k, v in query_params.items() if v} if query_params and len(query_params) > 0: - query_dict = QueryDict('', mutable=True) + query_dict = QueryDict("", mutable=True) for k in sorted(query_params.keys()): query_dict[k] = query_params[k] - url += ('?' + query_dict.urlencode(safe='/;:')) + url += "?" + query_dict.urlencode(safe="/;:") if request is not None: url = request.build_absolute_uri(url) return url def datetime_to_utc(date): """Returns datetime in UTC without timezone info Args: date (datetime.datetime): input datetime with timezone info Returns: datetime.datetime: datetime in UTC without timezone info """ if date.tzinfo: - return date.astimezone(tz.gettz('UTC')).replace(tzinfo=timezone.utc) + return date.astimezone(tz.gettz("UTC")).replace(tzinfo=timezone.utc) else: return date def parse_timestamp(timestamp): """Given a time or timestamp (as string), parse the result as UTC datetime. Returns: datetime.datetime: a timezone-aware datetime representing the parsed value or None if the parsing fails. Samples: - 2016-01-12 - 2016-01-12T09:19:12+0100 - Today is January 1, 2047 at 8:21:00AM - 1452591542 """ if not timestamp: return None try: date = date_parser.parse(timestamp, ignoretz=False, fuzzy=True) return datetime_to_utc(date) except Exception: try: return datetime.utcfromtimestamp(float(timestamp)).replace( - tzinfo=timezone.utc) + tzinfo=timezone.utc + ) except (ValueError, OverflowError) as e: raise BadInputExc(e) def shorten_path(path): """Shorten the given path: for each hash present, only return the first 8 characters followed by an ellipsis""" - sha256_re = r'([0-9a-f]{8})[0-9a-z]{56}' - sha1_re = r'([0-9a-f]{8})[0-9a-f]{32}' + sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}" + sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}" - ret = re.sub(sha256_re, r'\1...', path) - return re.sub(sha1_re, r'\1...', ret) + ret = re.sub(sha256_re, r"\1...", path) + return re.sub(sha1_re, r"\1...", ret) -def format_utc_iso_date(iso_date, fmt='%d %B %Y, %H:%M UTC'): +def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M UTC"): """Turns a string representation of an ISO 8601 date string to UTC and format it into a more human readable one. For instance, from the following input string: '2017-05-04T13:27:13+02:00' the following one is returned: '04 May 2017, 11:27 UTC'. Custom format string may also be provided as parameter Args: iso_date (str): a string representation of an ISO 8601 date fmt (str): optional date formatting string Returns: str: a formatted string representation of the input iso date """ if not iso_date: return iso_date date = parse_timestamp(iso_date) return date.strftime(fmt) def gen_path_info(path): """Function to generate path data navigation for use with a breadcrumb in the swh web ui. For instance, from a path /folder1/folder2/folder3, it returns the following list:: [{'name': 'folder1', 'path': 'folder1'}, {'name': 'folder2', 'path': 'folder1/folder2'}, {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] Args: path: a filesystem path Returns: list: a list of path data for navigation as illustrated above. """ path_info = [] if path: - sub_paths = path.strip('/').split('/') - path_from_root = '' + sub_paths = path.strip("/").split("/") + path_from_root = "" for p in sub_paths: - path_from_root += '/' + p - path_info.append({'name': p, - 'path': path_from_root.strip('/')}) + path_from_root += "/" + p + path_info.append({"name": p, "path": path_from_root.strip("/")}) return path_info def parse_rst(text, report_level=2): """ Parse a reStructuredText string with docutils. Args: text (str): string with reStructuredText markups in it report_level (int): level of docutils report messages to print (1 info 2 warning 3 error 4 severe 5 none) Returns: docutils.nodes.document: a parsed docutils document """ parser = docutils.parsers.rst.Parser() components = (docutils.parsers.rst.Parser,) settings = docutils.frontend.OptionParser( - components=components).get_default_values() + components=components + ).get_default_values() settings.report_level = report_level - document = docutils.utils.new_document('rst-doc', settings=settings) + document = docutils.utils.new_document("rst-doc", settings=settings) parser.parse(text, document) return document def get_client_ip(request): """ Return the client IP address from an incoming HTTP request. Args: request (django.http.HttpRequest): the incoming HTTP request Returns: str: The client IP address """ - x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: - ip = x_forwarded_for.split(',')[0] + ip = x_forwarded_for.split(",")[0] else: - ip = request.META.get('REMOTE_ADDR') + ip = request.META.get("REMOTE_ADDR") return ip def context_processor(request): """ Django context processor used to inject variables in all swh-web templates. """ config = get_config() - if request.user.is_authenticated and not hasattr(request.user, 'backend'): + if request.user.is_authenticated and not hasattr(request.user, "backend"): # To avoid django.template.base.VariableDoesNotExist errors # when rendering templates when standard Django user is logged in. - request.user.backend = 'django.contrib.auth.backends.ModelBackend' + request.user.backend = "django.contrib.auth.backends.ModelBackend" return { - 'swh_object_icons': swh_object_icons, - 'available_languages': None, - 'swh_client_config': config['client_config'], - 'oidc_enabled': bool(config['keycloak']['server_url']), + "swh_object_icons": swh_object_icons, + "available_languages": None, + "swh_client_config": config["client_config"], + "oidc_enabled": bool(config["keycloak"]["server_url"]), } class EnforceCSRFAuthentication(SessionAuthentication): """ Helper class to enforce CSRF validation on a DRF view when a user is not authenticated. """ def authenticate(self, request): - user = getattr(request._request, 'user', None) + user = getattr(request._request, "user", None) self.enforce_csrf(request) return (user, None) -def resolve_branch_alias(snapshot: Dict[str, Any], - branch: Optional[Dict[str, Any]] - ) -> Optional[Dict[str, Any]]: +def resolve_branch_alias( + snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] +) -> Optional[Dict[str, Any]]: """ Resolve branch alias in snapshot content. Args: snapshot: a full snapshot content branch: a branch alias contained in the snapshot Returns: The real snapshot branch that got aliased. """ - while branch and branch['target_type'] == 'alias': - if branch['target'] in snapshot['branches']: - branch = snapshot['branches'][branch['target']] + while branch and branch["target_type"] == "alias": + if branch["target"] in snapshot["branches"]: + branch = snapshot["branches"][branch["target"]] else: from swh.web.common import service + snp = service.lookup_snapshot( - snapshot['id'], branches_from=branch['target'], - branches_count=1) - if snp and branch['target'] in snp['branches']: - branch = snp['branches'][branch['target']] + snapshot["id"], branches_from=branch["target"], branches_count=1 + ) + if snp and branch["target"] in snp["branches"]: + branch = snp["branches"][branch["target"]] else: branch = None return branch class _NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] _HTML_WRITER = Writer() _HTML_WRITER.translator_class = _NoHeaderHTMLTranslator def rst_to_html(rst: str) -> str: """ Convert reStructuredText document into HTML. Args: rst: A string containing a reStructuredText document Returns: Body content of the produced HTML conversion. """ settings = { - 'initial_header_level': 2, + "initial_header_level": 2, } - pp = publish_parts(rst, writer=_HTML_WRITER, - settings_overrides=settings) + pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) return f'
{pp["html_body"]}
' def prettify_html(html: str) -> str: """ Prettify an HTML document. Args: html: Input HTML document Returns: The prettified HTML document """ - return BeautifulSoup(html, 'lxml').prettify() + return BeautifulSoup(html, "lxml").prettify() diff --git a/swh/web/config.py b/swh/web/config.py index 37113dbc..04f13574 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,188 +1,165 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict from swh.core import config from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { - 'allowed_hosts': ('list', []), - 'search': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://127.0.0.1:5010/', - 'timeout': 10, - }, - }), - 'storage': ('dict', { - 'cls': 'remote', - 'url': 'http://127.0.0.1:5002/', - 'timeout': 10, - }), - 'indexer_storage': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://127.0.0.1:5007/', - 'timeout': 1, - } - }), - 'log_dir': ('string', '/tmp/swh/log'), - 'debug': ('bool', False), - 'serve_assets': ('bool', False), - 'host': ('string', '127.0.0.1'), - 'port': ('int', 5004), - 'secret_key': ('string', 'development key'), + "allowed_hosts": ("list", []), + "search": ( + "dict", + {"cls": "remote", "args": {"url": "http://127.0.0.1:5010/", "timeout": 10,},}, + ), + "storage": ( + "dict", + {"cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10,}, + ), + "indexer_storage": ( + "dict", + {"cls": "remote", "args": {"url": "http://127.0.0.1:5007/", "timeout": 1,}}, + ), + "log_dir": ("string", "/tmp/swh/log"), + "debug": ("bool", False), + "serve_assets": ("bool", False), + "host": ("string", "127.0.0.1"), + "port": ("int", 5004), + "secret_key": ("string", "development key"), # do not display code highlighting for content > 1MB - 'content_display_max_size': ('int', 5 * 1024 * 1024), - 'snapshot_content_max_size': ('int', 1000), - 'throttling': ('dict', { - 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) - # development: in-memory cache so None - 'scopes': { - 'swh_api': { - 'limiter_rate': { - 'default': '120/h' + "content_display_max_size": ("int", 5 * 1024 * 1024), + "snapshot_content_max_size": ("int", 1000), + "throttling": ( + "dict", + { + "cache_uri": None, # production: memcached as cache (127.0.0.1:11211) + # development: in-memory cache so None + "scopes": { + "swh_api": { + "limiter_rate": {"default": "120/h"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_api_origin_search': { - 'limiter_rate': { - 'default': '10/m' + "swh_api_origin_search": { + "limiter_rate": {"default": "10/m"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_vault_cooking': { - 'limiter_rate': { - 'default': '120/h', - 'GET': '60/m' + "swh_vault_cooking": { + "limiter_rate": {"default": "120/h", "GET": "60/m"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_save_origin': { - 'limiter_rate': { - 'default': '120/h', - 'POST': '10/h' + "swh_save_origin": { + "limiter_rate": {"default": "120/h", "POST": "10/h"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_api_origin_visit_latest': { - 'limiter_rate': { - 'default': '700/m' + "swh_api_origin_visit_latest": { + "limiter_rate": {"default": "700/m"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'], }, - } - }), - 'vault': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://127.0.0.1:5005/', - } - }), - 'scheduler': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://127.0.0.1:5008/' - } - }), - 'development_db': ('string', os.path.join(SETTINGS_DIR, 'db.sqlite3')), - 'test_db': ('string', os.path.join(SETTINGS_DIR, 'testdb.sqlite3')), - 'production_db': ('string', '/var/lib/swh/web.sqlite3'), - 'deposit': ('dict', { - 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', - 'private_api_user': 'swhworker', - 'private_api_password': '' - }), - 'coverage_count_origins': ('bool', False), - 'e2e_tests_mode': ('bool', False), - 'es_workers_index_url': ('string', ''), - 'history_counters_url': ('string', 'https://stats.export.softwareheritage.org/history_counters.json'), # noqa - 'client_config': ('dict', {}), - 'keycloak': ('dict', { - 'server_url': '', - 'realm_name': '' - }), + }, + ), + "vault": ("dict", {"cls": "remote", "args": {"url": "http://127.0.0.1:5005/",}}), + "scheduler": ("dict", {"cls": "remote", "args": {"url": "http://127.0.0.1:5008/"}}), + "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")), + "test_db": ("string", os.path.join(SETTINGS_DIR, "testdb.sqlite3")), + "production_db": ("string", "/var/lib/swh/web.sqlite3"), + "deposit": ( + "dict", + { + "private_api_url": "https://deposit.softwareheritage.org/1/private/", + "private_api_user": "swhworker", + "private_api_password": "", + }, + ), + "coverage_count_origins": ("bool", False), + "e2e_tests_mode": ("bool", False), + "es_workers_index_url": ("string", ""), + "history_counters_url": ( + "string", + "https://stats.export.softwareheritage.org/history_counters.json", + ), + "client_config": ("dict", {}), + "keycloak": ("dict", {"server_url": "", "realm_name": ""}), } swhweb_config = {} # type: Dict[str, Any] -def get_config(config_file='web/web'): +def get_config(config_file="web/web"): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: - config_filename = os.environ.get('SWH_CONFIG_FILENAME') + config_filename = os.environ.get("SWH_CONFIG_FILENAME") if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) - config.prepare_folders(swhweb_config, 'log_dir') - if swhweb_config.get('search'): - swhweb_config['search'] = get_search(**swhweb_config['search']) + config.prepare_folders(swhweb_config, "log_dir") + if swhweb_config.get("search"): + swhweb_config["search"] = get_search(**swhweb_config["search"]) else: - swhweb_config['search'] = None - swhweb_config['storage'] = get_storage(**swhweb_config['storage']) - swhweb_config['vault'] = get_vault(**swhweb_config['vault']) - swhweb_config['indexer_storage'] = \ - get_indexer_storage(**swhweb_config['indexer_storage']) - swhweb_config['scheduler'] = get_scheduler( - **swhweb_config['scheduler']) + swhweb_config["search"] = None + swhweb_config["storage"] = get_storage(**swhweb_config["storage"]) + swhweb_config["vault"] = get_vault(**swhweb_config["vault"]) + swhweb_config["indexer_storage"] = get_indexer_storage( + **swhweb_config["indexer_storage"] + ) + swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) return swhweb_config def search(): """Return the current application's search. """ - return get_config()['search'] + return get_config()["search"] def storage(): """Return the current application's storage. """ - return get_config()['storage'] + return get_config()["storage"] def vault(): """Return the current application's vault. """ - return get_config()['vault'] + return get_config()["vault"] def indexer_storage(): """Return the current application's indexer storage. """ - return get_config()['indexer_storage'] + return get_config()["indexer_storage"] def scheduler(): """Return the current application's scheduler. """ - return get_config()['scheduler'] + return get_config()["scheduler"] diff --git a/swh/web/doc_config.py b/swh/web/doc_config.py index 65f5b170..c603cc94 100644 --- a/swh/web/doc_config.py +++ b/swh/web/doc_config.py @@ -1,73 +1,75 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import importlib.util import os from sphinxcontrib import httpdomain from sphinx.ext import autodoc # guard to avoid ImportError when running tests through sbuild # as there is no Debian package built for swh-docs -if importlib.util.find_spec('swh.docs'): +if importlib.util.find_spec("swh.docs"): from swh.docs.sphinx.conf import setup as orig_setup class SimpleDocumenter(autodoc.FunctionDocumenter): """ Custom autodoc directive to display a docstring unindented and without function signature header. """ + objtype = "simple" # ensure the priority is lesser than the base FunctionDocumenter # to avoid side effects with autodoc processing priority = -1 # do not indent the content content_indent = "" # do not add a header to the docstring def add_directive_header(self, sig): pass -_swh_web_base_url = 'https://archive.softwareheritage.org' -_swh_web_api_endpoint = 'api' +_swh_web_base_url = "https://archive.softwareheritage.org" +_swh_web_api_endpoint = "api" _swh_web_api_version = 1 -_swh_web_api_url = '%s/%s/%s/' % (_swh_web_base_url, - _swh_web_api_endpoint, - _swh_web_api_version) +_swh_web_api_url = "%s/%s/%s/" % ( + _swh_web_base_url, + _swh_web_api_endpoint, + _swh_web_api_version, +) -_swh_web_browse_endpoint = 'browse' -_swh_web_browse_url = '%s/%s/' % (_swh_web_base_url, - _swh_web_browse_endpoint) +_swh_web_browse_endpoint = "browse" +_swh_web_browse_url = "%s/%s/" % (_swh_web_base_url, _swh_web_browse_endpoint) def setup(app): orig_setup(app) app.add_autodocumenter(SimpleDocumenter) # set an environment variable indicating we are currently # building the swh-web documentation - os.environ['SWH_WEB_DOC_BUILD'] = '1' + os.environ["SWH_WEB_DOC_BUILD"] = "1" def customize_sphinx_conf(sphinx_conf): """ Utility function used to customize the sphinx doc build for swh-web globally (when building doc from swh-docs) or locally (when building doc from swh-web). Args: sphinx_conf (module): a reference to the sphinx conf.py module used to build the doc. """ # fix for sphinxcontrib.httpdomain 1.3 - if 'Link' not in httpdomain.HEADER_REFS: - httpdomain.HEADER_REFS['Link'] = httpdomain.IETFRef(5988, '5') - sphinx_conf.extlinks['swh_web'] = (_swh_web_base_url + '/%s', None) - sphinx_conf.extlinks['swh_web_api'] = (_swh_web_api_url + '%s', None) - sphinx_conf.extlinks['swh_web_browse'] = (_swh_web_browse_url + '%s', None) + if "Link" not in httpdomain.HEADER_REFS: + httpdomain.HEADER_REFS["Link"] = httpdomain.IETFRef(5988, "5") + sphinx_conf.extlinks["swh_web"] = (_swh_web_base_url + "/%s", None) + sphinx_conf.extlinks["swh_web_api"] = (_swh_web_api_url + "%s", None) + sphinx_conf.extlinks["swh_web_browse"] = (_swh_web_browse_url + "%s", None) sphinx_conf.setup = setup diff --git a/swh/web/manage.py b/swh/web/manage.py index 4ab9fdd1..e8d69688 100755 --- a/swh/web/manage.py +++ b/swh/web/manage.py @@ -1,46 +1,44 @@ #!/usr/bin/env python # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import sys from swh.web import config if __name__ == "__main__": swh_web_config = config.get_config() # the serving of static assets in development mode is handled # in swh/web/urls.py, we pass the nostatic options to runserver # in order to have gzip compression enabled. - swh_web_config['serve_assets'] = '--nostatic' in sys.argv + swh_web_config["serve_assets"] = "--nostatic" in sys.argv # import root urls module for swh-web before running the django dev server # in order to ensure it will be automatically reloaded when source files # are modified (as django autoreload feature only works if the modules are # in sys.modules) try: - from swh.web import urls # noqa + from swh.web import urls # noqa except Exception: pass try: - from django.core.management.commands.runserver import ( - Command as runserver - ) + from django.core.management.commands.runserver import Command as runserver from django.core.management import execute_from_command_line except ImportError: # The above import may fail for some other reason. Ensure that the # issue is really that Django is missing to avoid masking other # exceptions on Python 2. try: - import django # noqa + import django # noqa except ImportError: raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?" ) raise - runserver.default_port = swh_web_config['port'] - runserver.default_addr = swh_web_config['host'] + runserver.default_port = swh_web_config["port"] + runserver.default_addr = swh_web_config["host"] execute_from_command_line(sys.argv) diff --git a/swh/web/misc/badges.py b/swh/web/misc/badges.py index 596d38c9..23947cee 100644 --- a/swh/web/misc/badges.py +++ b/swh/web/misc/badges.py @@ -1,170 +1,167 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from base64 import b64encode from typing import cast, Optional from django.conf.urls import url from django.contrib.staticfiles import finders from django.http import HttpResponse, HttpRequest from pybadges import badge from swh.model.exceptions import ValidationError from swh.model.identifiers import ( - persistent_identifier, parse_persistent_identifier, - CONTENT, DIRECTORY, ORIGIN, RELEASE, REVISION, SNAPSHOT + persistent_identifier, + parse_persistent_identifier, + CONTENT, + DIRECTORY, + ORIGIN, + RELEASE, + REVISION, + SNAPSHOT, ) from swh.web.common import service from swh.web.common.exc import BadInputExc, NotFoundExc from swh.web.common.identifiers import resolve_swh_persistent_id from swh.web.common.utils import reverse -_orange = '#f36a24' -_blue = '#0172b2' -_red = '#cd5741' +_orange = "#f36a24" +_blue = "#0172b2" +_red = "#cd5741" _swh_logo_data = None _badge_config = { - CONTENT: { - 'color': _blue, - 'title': 'Archived source file', - }, - DIRECTORY: { - 'color': _blue, - 'title': 'Archived source tree', - }, - ORIGIN: { - 'color': _orange, - 'title': 'Archived software repository', - }, - RELEASE: { - 'color': _blue, - 'title': 'Archived software release', - }, - REVISION: { - 'color': _blue, - 'title': 'Archived commit', - }, - SNAPSHOT: { - 'color': _blue, - 'title': 'Archived software repository snapshot', - }, - 'error': { - 'color': _red, - 'title': 'An error occurred when generating the badge' - } + CONTENT: {"color": _blue, "title": "Archived source file",}, + DIRECTORY: {"color": _blue, "title": "Archived source tree",}, + ORIGIN: {"color": _orange, "title": "Archived software repository",}, + RELEASE: {"color": _blue, "title": "Archived software release",}, + REVISION: {"color": _blue, "title": "Archived commit",}, + SNAPSHOT: {"color": _blue, "title": "Archived software repository snapshot",}, + "error": {"color": _red, "title": "An error occurred when generating the badge"}, } def _get_logo_data() -> str: """ Get data-URI for Software Heritage SVG logo to embed it in the generated badges. """ global _swh_logo_data if _swh_logo_data is None: - swh_logo_path = cast(str, finders.find('img/swh-logo-white.svg')) - with open(swh_logo_path, 'rb') as swh_logo_file: - _swh_logo_data = ('data:image/svg+xml;base64,%s' % - b64encode(swh_logo_file.read()).decode('ascii')) + swh_logo_path = cast(str, finders.find("img/swh-logo-white.svg")) + with open(swh_logo_path, "rb") as swh_logo_file: + _swh_logo_data = "data:image/svg+xml;base64,%s" % b64encode( + swh_logo_file.read() + ).decode("ascii") return _swh_logo_data -def _swh_badge(request: HttpRequest, object_type: str, object_id: str, - object_pid: Optional[str] = '') -> HttpResponse: +def _swh_badge( + request: HttpRequest, + object_type: str, + object_id: str, + object_pid: Optional[str] = "", +) -> HttpResponse: """ Generate a Software Heritage badge for a given object type and id. Args: request: input http request object_type: The type of swh object to generate a badge for, either *content*, *directory*, *revision*, *release*, *origin* or *snapshot* object_id: The id of the swh object, either an url for origin type or a *sha1* for other object types object_pid: If provided, the object persistent identifier will not be recomputed Returns: HTTP response with content type *image/svg+xml* containing the SVG badge data. If the provided parameters are invalid, HTTP 400 status code will be returned. If the object can not be found in the archive, HTTP 404 status code will be returned. """ - left_text = 'error' + left_text = "error" whole_link = None try: if object_type == ORIGIN: - service.lookup_origin({'url': object_id}) - right_text = 'repository' - whole_link = reverse('browse-origin', - url_args={'origin_url': object_id}) + service.lookup_origin({"url": object_id}) + right_text = "repository" + whole_link = reverse("browse-origin", url_args={"origin_url": object_id}) else: # when pid is provided, object type and id will be parsed # from it if object_pid: parsed_pid = parse_persistent_identifier(object_pid) object_type = parsed_pid.object_type object_id = parsed_pid.object_id swh_object = service.lookup_object(object_type, object_id) if object_pid: right_text = object_pid else: right_text = persistent_identifier(object_type, object_id) - whole_link = resolve_swh_persistent_id(right_text)['browse_url'] + whole_link = resolve_swh_persistent_id(right_text)["browse_url"] # remove pid metadata if any for badge text if object_pid: - right_text = right_text.split(';')[0] + right_text = right_text.split(";")[0] # use release name for badge text if object_type == RELEASE: - right_text = 'release %s' % swh_object['name'] - left_text = 'archived' + right_text = "release %s" % swh_object["name"] + left_text = "archived" except (BadInputExc, ValidationError): right_text = f'invalid {object_type if object_type else "object"} id' - object_type = 'error' + object_type = "error" except NotFoundExc: right_text = f'{object_type if object_type else "object"} not found' - object_type = 'error' + object_type = "error" - badge_data = badge(left_text=left_text, - right_text=right_text, - right_color=_badge_config[object_type]['color'], - whole_link=request.build_absolute_uri(whole_link), - whole_title=_badge_config[object_type]['title'], - logo=_get_logo_data(), - embed_logo=True) + badge_data = badge( + left_text=left_text, + right_text=right_text, + right_color=_badge_config[object_type]["color"], + whole_link=request.build_absolute_uri(whole_link), + whole_title=_badge_config[object_type]["title"], + logo=_get_logo_data(), + embed_logo=True, + ) - return HttpResponse(badge_data, content_type='image/svg+xml') + return HttpResponse(badge_data, content_type="image/svg+xml") def _swh_badge_pid(request: HttpRequest, object_pid: str) -> HttpResponse: """ Generate a Software Heritage badge for a given object persistent identifier. Args: request (django.http.HttpRequest): input http request object_pid (str): A swh object persistent identifier Returns: django.http.HttpResponse: An http response with content type *image/svg+xml* containing the SVG badge data. If any error occurs, a status code of 400 will be returned. """ - return _swh_badge(request, '', '', object_pid) + return _swh_badge(request, "", "", object_pid) urlpatterns = [ - url(r'^badge/(?P[a-z]+)/(?P.+)/$', _swh_badge, - name='swh-badge'), - url(r'^badge/(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', - _swh_badge_pid, name='swh-badge-pid'), + url( + r"^badge/(?P[a-z]+)/(?P.+)/$", + _swh_badge, + name="swh-badge", + ), + url( + r"^badge/(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$", + _swh_badge_pid, + name="swh-badge-pid", + ), ] diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py index d9ad8b9c..22547d85 100644 --- a/swh/web/misc/coverage.py +++ b/swh/web/misc/coverage.py @@ -1,199 +1,198 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf.urls import url from django.core.cache import caches from django.http import HttpResponse from django.shortcuts import render from django.views.decorators.cache import never_cache from django.views.decorators.clickjacking import xframe_options_exempt from swh.web.common import service from swh.web.common.exc import handle_view_exception from swh.web.config import get_config # Current coverage list of the archive # TODO: Retrieve that list dynamically instead of hardcoding it _code_providers = [ { - 'provider_id': 'bitbucket', - 'provider_url': 'https://bitbucket.org/', - 'provider_logo': 'img/logos/bitbucket.png', - 'provider_info': 'public repositories from Bitbucket ' - '(continuously archived)', - 'origin_url_regexp': '^https://bitbucket.org/', - 'origin_types': 'repositories', + "provider_id": "bitbucket", + "provider_url": "https://bitbucket.org/", + "provider_logo": "img/logos/bitbucket.png", + "provider_info": "public repositories from Bitbucket " + "(continuously archived)", + "origin_url_regexp": "^https://bitbucket.org/", + "origin_types": "repositories", }, { - 'provider_id': 'cran', - 'provider_url': 'https://cran.r-project.org/', - 'provider_logo': 'img/logos/cran.svg', - 'provider_info': 'source packages from The Comprehensive R Archive ' - 'Network (continuously archived)', - 'origin_url_regexp': '^https://cran.r-project.org/', - 'origin_types': 'packages', + "provider_id": "cran", + "provider_url": "https://cran.r-project.org/", + "provider_logo": "img/logos/cran.svg", + "provider_info": "source packages from The Comprehensive R Archive " + "Network (continuously archived)", + "origin_url_regexp": "^https://cran.r-project.org/", + "origin_types": "packages", }, { - 'provider_id': 'debian', - 'provider_url': 'https://www.debian.org/', - 'provider_logo': 'img/logos/debian.png', - 'provider_info': 'source packages from the Debian distribution ' - '(continuously archived)', - 'origin_url_regexp': '^deb://', - 'origin_types': 'packages', + "provider_id": "debian", + "provider_url": "https://www.debian.org/", + "provider_logo": "img/logos/debian.png", + "provider_info": "source packages from the Debian distribution " + "(continuously archived)", + "origin_url_regexp": "^deb://", + "origin_types": "packages", }, { - 'provider_id': 'framagit', - 'provider_url': 'https://framagit.org/', - 'provider_logo': 'img/logos/framagit.png', - 'provider_info': 'public repositories from Framagit ' - '(continuously archived)', - 'origin_url_regexp': '^https://framagit.org/', - 'origin_types': 'repositories', + "provider_id": "framagit", + "provider_url": "https://framagit.org/", + "provider_logo": "img/logos/framagit.png", + "provider_info": "public repositories from Framagit " "(continuously archived)", + "origin_url_regexp": "^https://framagit.org/", + "origin_types": "repositories", }, { - 'provider_id': 'github', - 'provider_url': 'https://github.com', - 'provider_logo': 'img/logos/github.png', - 'provider_info': 'public repositories from GitHub ' - '(continuously archived)', - 'origin_url_regexp': '^https://github.com/', - 'origin_types': 'repositories', + "provider_id": "github", + "provider_url": "https://github.com", + "provider_logo": "img/logos/github.png", + "provider_info": "public repositories from GitHub " "(continuously archived)", + "origin_url_regexp": "^https://github.com/", + "origin_types": "repositories", }, { - 'provider_id': 'gitlab', - 'provider_url': 'https://gitlab.com', - 'provider_logo': 'img/logos/gitlab.svg', - 'provider_info': 'public repositories from GitLab ' - '(continuously archived)', - 'origin_url_regexp': '^https://gitlab.com/', - 'origin_types': 'repositories', + "provider_id": "gitlab", + "provider_url": "https://gitlab.com", + "provider_logo": "img/logos/gitlab.svg", + "provider_info": "public repositories from GitLab " "(continuously archived)", + "origin_url_regexp": "^https://gitlab.com/", + "origin_types": "repositories", }, { - 'provider_id': 'gitorious', - 'provider_url': 'https://gitorious.org/', - 'provider_logo': 'img/logos/gitorious.png', - 'provider_info': 'public repositories from the former Gitorious code ' - 'hosting service', - 'origin_url_regexp': '^https://gitorious.org/', - 'origin_types': 'repositories', + "provider_id": "gitorious", + "provider_url": "https://gitorious.org/", + "provider_logo": "img/logos/gitorious.png", + "provider_info": "public repositories from the former Gitorious code " + "hosting service", + "origin_url_regexp": "^https://gitorious.org/", + "origin_types": "repositories", }, { - 'provider_id': 'googlecode', - 'provider_url': 'https://code.google.com/archive/', - 'provider_logo': 'img/logos/googlecode.png', - 'provider_info': 'public repositories from the former Google Code ' - 'project hosting service', - 'origin_url_regexp': '^http.*.googlecode.com/', - 'origin_types': 'repositories', + "provider_id": "googlecode", + "provider_url": "https://code.google.com/archive/", + "provider_logo": "img/logos/googlecode.png", + "provider_info": "public repositories from the former Google Code " + "project hosting service", + "origin_url_regexp": "^http.*.googlecode.com/", + "origin_types": "repositories", }, { - 'provider_id': 'gnu', - 'provider_url': 'https://www.gnu.org', - 'provider_logo': 'img/logos/gnu.png', - 'provider_info': 'releases from the GNU project (as of August 2015)', - 'origin_url_regexp': '^rsync://ftp.gnu.org/', - 'origin_types': 'releases', + "provider_id": "gnu", + "provider_url": "https://www.gnu.org", + "provider_logo": "img/logos/gnu.png", + "provider_info": "releases from the GNU project (as of August 2015)", + "origin_url_regexp": "^rsync://ftp.gnu.org/", + "origin_types": "releases", }, { - 'provider_id': 'hal', - 'provider_url': 'https://hal.archives-ouvertes.fr/', - 'provider_logo': 'img/logos/hal.png', - 'provider_info': 'scientific software source code deposited in the ' - 'open archive HAL', - 'origin_url_regexp': '^https://hal.archives-ouvertes.fr/', - 'origin_types': 'deposits', - + "provider_id": "hal", + "provider_url": "https://hal.archives-ouvertes.fr/", + "provider_logo": "img/logos/hal.png", + "provider_info": "scientific software source code deposited in the " + "open archive HAL", + "origin_url_regexp": "^https://hal.archives-ouvertes.fr/", + "origin_types": "deposits", }, { - 'provider_id': 'inria', - 'provider_url': 'https://gitlab.inria.fr', - 'provider_logo': 'img/logos/inria.jpg', - 'provider_info': 'public repositories from Inria GitLab ' - '(continuously archived)', - 'origin_url_regexp': '^https://gitlab.inria.fr/', - 'origin_types': 'repositories', + "provider_id": "inria", + "provider_url": "https://gitlab.inria.fr", + "provider_logo": "img/logos/inria.jpg", + "provider_info": "public repositories from Inria GitLab " + "(continuously archived)", + "origin_url_regexp": "^https://gitlab.inria.fr/", + "origin_types": "repositories", }, { - 'provider_id': 'npm', - 'provider_url': 'https://www.npmjs.com/', - 'provider_logo': 'img/logos/npm.png', - 'provider_info': 'public packages from the package registry for ' - 'javascript (continuously archived)', - 'origin_url_regexp': '^https://www.npmjs.com/', - 'origin_types': 'packages', + "provider_id": "npm", + "provider_url": "https://www.npmjs.com/", + "provider_logo": "img/logos/npm.png", + "provider_info": "public packages from the package registry for " + "javascript (continuously archived)", + "origin_url_regexp": "^https://www.npmjs.com/", + "origin_types": "packages", }, { - 'provider_id': 'pypi', - 'provider_url': 'https://pypi.org', - 'provider_logo': 'img/logos/pypi.svg', - 'provider_info': 'source packages from the Python Packaging Index ' - '(continuously archived)', - 'origin_url_regexp': '^https://pypi.org/', - 'origin_types': 'packages', + "provider_id": "pypi", + "provider_url": "https://pypi.org", + "provider_logo": "img/logos/pypi.svg", + "provider_info": "source packages from the Python Packaging Index " + "(continuously archived)", + "origin_url_regexp": "^https://pypi.org/", + "origin_types": "packages", }, ] @xframe_options_exempt def _swh_coverage(request): - count_origins = get_config()['coverage_count_origins'] - return render(request, 'misc/coverage.html', - {'providers': _code_providers, - 'count_origins': count_origins}) + count_origins = get_config()["coverage_count_origins"] + return render( + request, + "misc/coverage.html", + {"providers": _code_providers, "count_origins": count_origins}, + ) @never_cache def _swh_coverage_count(request): """Internal browse endpoint to count the number of origins associated to each code provider declared in the archive coverage list. As this operation takes some times, we execute it once per day and cache its results to database. The cached origin counts are then served. Cache management is handled in the implementation to avoid sending the same count query twice to the storage database. """ try: - cache = caches['db_cache'] + cache = caches["db_cache"] results = [] for code_provider in _code_providers: - provider_id = code_provider['provider_id'] - url_regexp = code_provider['origin_url_regexp'] - cache_key = '%s_origins_count' % provider_id - prev_cache_key = '%s_origins_prev_count' % provider_id + provider_id = code_provider["provider_id"] + url_regexp = code_provider["origin_url_regexp"] + cache_key = "%s_origins_count" % provider_id + prev_cache_key = "%s_origins_prev_count" % provider_id # get cached origin count origin_count = cache.get(cache_key, -2) # cache entry has expired or does not exist if origin_count == -2: # mark the origin count as processing - cache.set(cache_key, -1, timeout=10*60) + cache.set(cache_key, -1, timeout=10 * 60) # execute long count query - origin_count = service.storage.origin_count(url_regexp, - regexp=True) + origin_count = service.storage.origin_count(url_regexp, regexp=True) # cache count result - cache.set(cache_key, origin_count, timeout=24*60*60) + cache.set(cache_key, origin_count, timeout=24 * 60 * 60) cache.set(prev_cache_key, origin_count, timeout=None) # origin count is currently processing elif origin_count == -1: # return previous count if it exists origin_count = cache.get(prev_cache_key, -1) - results.append({ - 'provider_id': provider_id, - 'origin_count': origin_count, - 'origin_types': code_provider['origin_types'] - }) + results.append( + { + "provider_id": provider_id, + "origin_count": origin_count, + "origin_types": code_provider["origin_types"], + } + ) results = json.dumps(results) except Exception as exc: return handle_view_exception(request, exc, html_response=False) - return HttpResponse(results, content_type='application/json') + return HttpResponse(results, content_type="application/json") urlpatterns = [ - url(r'^coverage/$', _swh_coverage, name='swh-coverage'), - url(r'^coverage/count/$', _swh_coverage_count, name='swh-coverage-count'), + url(r"^coverage/$", _swh_coverage, name="swh-coverage"), + url(r"^coverage/count/$", _swh_coverage_count, name="swh-coverage-count"), ] diff --git a/swh/web/misc/metrics.py b/swh/web/misc/metrics.py index a7f5c7d1..4667f169 100644 --- a/swh/web/misc/metrics.py +++ b/swh/web/misc/metrics.py @@ -1,19 +1,20 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpResponse from prometheus_client.exposition import generate_latest, CONTENT_TYPE_LATEST from swh.web.common.origin_save import compute_save_requests_metrics from swh.web.common.utils import SWH_WEB_METRICS_REGISTRY def prometheus_metrics(request): compute_save_requests_metrics() return HttpResponse( content=generate_latest(registry=SWH_WEB_METRICS_REGISTRY), - content_type=CONTENT_TYPE_LATEST) + content_type=CONTENT_TYPE_LATEST, + ) diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py index 2ec25bf5..313d02b7 100644 --- a/swh/web/misc/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -1,108 +1,117 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf.urls import url from django.core.paginator import Paginator -from django.http import ( - HttpResponse, HttpResponseForbidden, HttpResponseServerError -) +from django.http import HttpResponse, HttpResponseForbidden, HttpResponseServerError from django.shortcuts import render from rest_framework.decorators import api_view, authentication_classes from swh.web.api.throttling import throttle_scope from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( - create_save_origin_request, get_savable_visit_types, - get_save_origin_requests_from_queryset + create_save_origin_request, + get_savable_visit_types, + get_save_origin_requests_from_queryset, ) from swh.web.common.utils import EnforceCSRFAuthentication def _origin_save_view(request): - return render(request, 'misc/origin-save.html', - {'heading': ('Request the saving of a software origin into ' - 'the archive')}) + return render( + request, + "misc/origin-save.html", + {"heading": ("Request the saving of a software origin into " "the archive")}, + ) -@api_view(['POST']) -@authentication_classes((EnforceCSRFAuthentication, )) -@throttle_scope('swh_save_origin') +@api_view(["POST"]) +@authentication_classes((EnforceCSRFAuthentication,)) +@throttle_scope("swh_save_origin") def _origin_save_request(request, visit_type, origin_url): """ This view is called through AJAX from the save code now form of swh-web. We use DRF here as we want to rate limit the number of submitted requests per user to avoid being possibly flooded by bots. """ try: - response = json.dumps(create_save_origin_request(visit_type, - origin_url), - separators=(',', ': ')) - return HttpResponse(response, content_type='application/json') + response = json.dumps( + create_save_origin_request(visit_type, origin_url), separators=(",", ": ") + ) + return HttpResponse(response, content_type="application/json") except ForbiddenExc as exc: - return HttpResponseForbidden(json.dumps({'detail': str(exc)}), - content_type='application/json') + return HttpResponseForbidden( + json.dumps({"detail": str(exc)}), content_type="application/json" + ) except Exception as exc: - return HttpResponseServerError(json.dumps({'detail': str(exc)}), - content_type='application/json') + return HttpResponseServerError( + json.dumps({"detail": str(exc)}), content_type="application/json" + ) def _visit_save_types_list(request): - visit_types = json.dumps(get_savable_visit_types(), - separators=(',', ': ')) - return HttpResponse(visit_types, content_type='application/json') + visit_types = json.dumps(get_savable_visit_types(), separators=(",", ": ")) + return HttpResponse(visit_types, content_type="application/json") def _origin_save_requests_list(request, status): - if status != 'all': + if status != "all": save_requests = SaveOriginRequest.objects.filter(status=status) else: save_requests = SaveOriginRequest.objects.all() table_data = {} - table_data['recordsTotal'] = save_requests.count() - table_data['draw'] = int(request.GET['draw']) + table_data["recordsTotal"] = save_requests.count() + table_data["draw"] = int(request.GET["draw"]) - search_value = request.GET['search[value]'] + search_value = request.GET["search[value]"] - column_order = request.GET['order[0][column]'] - field_order = request.GET['columns[%s][name]' % column_order] - order_dir = request.GET['order[0][dir]'] - if order_dir == 'desc': - field_order = '-' + field_order + column_order = request.GET["order[0][column]"] + field_order = request.GET["columns[%s][name]" % column_order] + order_dir = request.GET["order[0][dir]"] + if order_dir == "desc": + field_order = "-" + field_order save_requests = save_requests.order_by(field_order) - length = int(request.GET['length']) - page = int(request.GET['start']) / length + 1 + length = int(request.GET["length"]) + page = int(request.GET["start"]) / length + 1 save_requests = get_save_origin_requests_from_queryset(save_requests) if search_value: - save_requests = \ - [sr for sr in save_requests - if search_value.lower() in sr['save_request_status'].lower() - or search_value.lower() in sr['save_task_status'].lower() - or search_value.lower() in sr['visit_type'].lower() - or search_value.lower() in sr['origin_url'].lower()] - - table_data['recordsFiltered'] = len(save_requests) + save_requests = [ + sr + for sr in save_requests + if search_value.lower() in sr["save_request_status"].lower() + or search_value.lower() in sr["save_task_status"].lower() + or search_value.lower() in sr["visit_type"].lower() + or search_value.lower() in sr["origin_url"].lower() + ] + + table_data["recordsFiltered"] = len(save_requests) paginator = Paginator(save_requests, length) - table_data['data'] = paginator.page(page).object_list - table_data_json = json.dumps(table_data, separators=(',', ': ')) - return HttpResponse(table_data_json, content_type='application/json') + table_data["data"] = paginator.page(page).object_list + table_data_json = json.dumps(table_data, separators=(",", ": ")) + return HttpResponse(table_data_json, content_type="application/json") urlpatterns = [ - url(r'^save/$', _origin_save_view, name='origin-save'), - url(r'^save/(?P.+)/url/(?P.+)/$', - _origin_save_request, name='origin-save-request'), - url(r'^save/types/list/$', _visit_save_types_list, - name='origin-save-types-list'), - url(r'^save/requests/list/(?P.+)/$', _origin_save_requests_list, - name='origin-save-requests-list'), + url(r"^save/$", _origin_save_view, name="origin-save"), + url( + r"^save/(?P.+)/url/(?P.+)/$", + _origin_save_request, + name="origin-save-request", + ), + url(r"^save/types/list/$", _visit_save_types_list, name="origin-save-types-list"), + url( + r"^save/requests/list/(?P.+)/$", + _origin_save_requests_list, + name="origin-save-requests-list", + ), ] diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py index 413423d8..a128744b 100644 --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -1,83 +1,102 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import requests import sentry_sdk from django.conf.urls import url, include from django.contrib.staticfiles import finders from django.http import HttpResponse from django.shortcuts import render from swh.web.common import service from swh.web.config import get_config from swh.web.misc.metrics import prometheus_metrics def _jslicenses(request): - jslicenses_file = finders.find('jssources/jslicenses.json') + jslicenses_file = finders.find("jssources/jslicenses.json") jslicenses_data = json.load(open(jslicenses_file)) - jslicenses_data = sorted(jslicenses_data.items(), - key=lambda item: item[0].split('/')[-1]) - return render(request, "misc/jslicenses.html", - {'jslicenses_data': jslicenses_data}) + jslicenses_data = sorted( + jslicenses_data.items(), key=lambda item: item[0].split("/")[-1] + ) + return render(request, "misc/jslicenses.html", {"jslicenses_data": jslicenses_data}) def _stat_counters(request): stat = service.stat_counters() - url = get_config()['history_counters_url'] - stat_counters_history = 'null' + url = get_config()["history_counters_url"] + stat_counters_history = "null" if url: try: response = requests.get(url, timeout=5) stat_counters_history = response.text except Exception as exc: sentry_sdk.capture_exception(exc) json_data = '{"stat_counters": %s, "stat_counters_history": %s}' % ( - json.dumps(stat), stat_counters_history) - return HttpResponse(json_data, content_type='application/json') + json.dumps(stat), + stat_counters_history, + ) + return HttpResponse(json_data, content_type="application/json") urlpatterns = [ - url(r'^', include('swh.web.misc.coverage')), - url(r'^jslicenses/$', _jslicenses, name='jslicenses'), - url(r'^', include('swh.web.misc.origin_save')), - url(r'^stat_counters/', _stat_counters, name='stat-counters'), - url(r'^', include('swh.web.misc.badges')), - url(r'^metrics/prometheus/$', prometheus_metrics, - name='metrics-prometheus'), + url(r"^", include("swh.web.misc.coverage")), + url(r"^jslicenses/$", _jslicenses, name="jslicenses"), + url(r"^", include("swh.web.misc.origin_save")), + url(r"^stat_counters/", _stat_counters, name="stat-counters"), + url(r"^", include("swh.web.misc.badges")), + url(r"^metrics/prometheus/$", prometheus_metrics, name="metrics-prometheus"), ] # when running end to end tests trough cypress, declare some extra # endpoints to provide input data for some of those tests -if get_config()['e2e_tests_mode']: +if get_config()["e2e_tests_mode"]: from swh.web.tests.views import ( get_content_code_data_by_ext, get_content_other_data_by_ext, get_content_code_data_all_exts, get_content_code_data_by_filename, get_content_code_data_all_filenames, - ) # noqa + ) + urlpatterns.append( - url(r'^tests/data/content/code/extension/(?P.+)/$', + url( + r"^tests/data/content/code/extension/(?P.+)/$", get_content_code_data_by_ext, - name='tests-content-code-extension')) + name="tests-content-code-extension", + ) + ) urlpatterns.append( - url(r'^tests/data/content/other/extension/(?P.+)/$', + url( + r"^tests/data/content/other/extension/(?P.+)/$", get_content_other_data_by_ext, - name='tests-content-other-extension')) - urlpatterns.append(url(r'^tests/data/content/code/extensions/$', - get_content_code_data_all_exts, - name='tests-content-code-extensions')) + name="tests-content-other-extension", + ) + ) + urlpatterns.append( + url( + r"^tests/data/content/code/extensions/$", + get_content_code_data_all_exts, + name="tests-content-code-extensions", + ) + ) urlpatterns.append( - url(r'^tests/data/content/code/filename/(?P.+)/$', + url( + r"^tests/data/content/code/filename/(?P.+)/$", get_content_code_data_by_filename, - name='tests-content-code-filename')) - urlpatterns.append(url(r'^tests/data/content/code/filenames/$', - get_content_code_data_all_filenames, - name='tests-content-code-filenames')) + name="tests-content-code-filename", + ) + ) + urlpatterns.append( + url( + r"^tests/data/content/code/filenames/$", + get_content_code_data_all_filenames, + name="tests-content-code-filenames", + ) + ) diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 110b6e5b..5488828c 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,303 +1,284 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os import sys from typing import Any, Dict from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = swh_web_config['secret_key'] +SECRET_KEY = swh_web_config["secret_key"] # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = swh_web_config['debug'] -DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] +DEBUG = swh_web_config["debug"] +DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config["debug"] -ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] +ALLOWED_HOSTS = ["127.0.0.1", "localhost"] + swh_web_config["allowed_hosts"] # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'swh.web.common', - 'swh.web.api', - 'swh.web.browse', - 'webpack_loader', - 'django_js_reverse', - 'corsheaders', + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", + "swh.web.common", + "swh.web.api", + "swh.web.browse", + "webpack_loader", + "django_js_reverse", + "corsheaders", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'swh.web.auth.middlewares.OIDCSessionRefreshMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'swh.web.common.middlewares.ThrottlingHeadersMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "corsheaders.middleware.CorsMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "swh.web.auth.middlewares.OIDCSessionRefreshMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "swh.web.common.middlewares.ThrottlingHeadersMiddleware", ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. -if swh_web_config['serve_assets']: - MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware') +if swh_web_config["serve_assets"]: + MIDDLEWARE.insert(0, "django.middleware.gzip.GZipMiddleware") -ROOT_URLCONF = 'swh.web.urls' +ROOT_URLCONF = "swh.web.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(PROJECT_DIR, "../templates")], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', - 'swh.web.common.utils.context_processor' + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(PROJECT_DIR, "../templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + "swh.web.common.utils.context_processor", ], - 'libraries': { - 'swh_templatetags': 'swh.web.common.swh_templatetags', - }, + "libraries": {"swh_templatetags": "swh.web.common.swh_templatetags",}, }, }, ] DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': swh_web_config['development_db'], + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": swh_web_config["development_db"], } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa }, + {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, + {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, + {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" # static folder location when swh-web has been installed with pip -STATIC_DIR = os.path.join(sys.prefix, 'share/swh/web/static') +STATIC_DIR = os.path.join(sys.prefix, "share/swh/web/static") if not os.path.exists(STATIC_DIR): # static folder location when developping swh-web - STATIC_DIR = os.path.join(PROJECT_DIR, '../../../static') + STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static") STATICFILES_DIRS = [STATIC_DIR] -INTERNAL_IPS = ['127.0.0.1'] +INTERNAL_IPS = ["127.0.0.1"] throttle_rates = {} -http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'] +http_requests = ["GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"] -throttling = swh_web_config['throttling'] -for limiter_scope, limiter_conf in throttling['scopes'].items(): - if 'default' in limiter_conf['limiter_rate']: - throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default'] +throttling = swh_web_config["throttling"] +for limiter_scope, limiter_conf in throttling["scopes"].items(): + if "default" in limiter_conf["limiter_rate"]: + throttle_rates[limiter_scope] = limiter_conf["limiter_rate"]["default"] # for backward compatibility else: - throttle_rates[limiter_scope] = limiter_conf['limiter_rate'] + throttle_rates[limiter_scope] = limiter_conf["limiter_rate"] # register sub scopes specific for HTTP request types for http_request in http_requests: - if http_request in limiter_conf['limiter_rate']: - throttle_rates[limiter_scope + '_' + http_request.lower()] = \ - limiter_conf['limiter_rate'][http_request] + if http_request in limiter_conf["limiter_rate"]: + throttle_rates[limiter_scope + "_" + http_request.lower()] = limiter_conf[ + "limiter_rate" + ][http_request] REST_FRAMEWORK: Dict[str, Any] = { - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'swh.web.api.renderers.YAMLRenderer', - 'rest_framework.renderers.TemplateHTMLRenderer' - ), - 'DEFAULT_THROTTLE_CLASSES': ( - 'swh.web.api.throttling.SwhWebRateThrottle', + "DEFAULT_RENDERER_CLASSES": ( + "rest_framework.renderers.JSONRenderer", + "swh.web.api.renderers.YAMLRenderer", + "rest_framework.renderers.TemplateHTMLRenderer", ), - 'DEFAULT_THROTTLE_RATES': throttle_rates, - 'DEFAULT_AUTHENTICATION_CLASSES': [ - 'rest_framework.authentication.SessionAuthentication', - 'swh.web.auth.backends.OIDCBearerTokenAuthentication', + "DEFAULT_THROTTLE_CLASSES": ("swh.web.api.throttling.SwhWebRateThrottle",), + "DEFAULT_THROTTLE_RATES": throttle_rates, + "DEFAULT_AUTHENTICATION_CLASSES": [ + "rest_framework.authentication.SessionAuthentication", + "swh.web.auth.backends.OIDCBearerTokenAuthentication", ], } LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', - }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', - }, + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": {"()": "django.utils.log.RequireDebugFalse",}, + "require_debug_true": {"()": "django.utils.log.RequireDebugTrue",}, }, - 'formatters': { - 'request': { - 'format': '[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s', # noqa - 'datefmt': "%d/%b/%Y %H:%M:%S" + "formatters": { + "request": { + "format": "[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s", + "datefmt": "%d/%b/%Y %H:%M:%S", }, - 'simple': { - 'format': '[%(asctime)s] [%(levelname)s] %(message)s', - 'datefmt': "%d/%b/%Y %H:%M:%S" + "simple": { + "format": "[%(asctime)s] [%(levelname)s] %(message)s", + "datefmt": "%d/%b/%Y %H:%M:%S", }, - 'verbose': { - 'format': '[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s - %(message)s', # noqa - 'datefmt': "%d/%b/%Y %H:%M:%S" + "verbose": { + "format": ( + "[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s " + "- %(message)s" + ), + "datefmt": "%d/%b/%Y %H:%M:%S", }, }, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'simple' - }, - 'file': { - 'level': 'WARNING', - 'filters': ['require_debug_false'], - 'class': 'logging.FileHandler', - 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), - 'formatter': 'simple' + "handlers": { + "console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "simple", }, - 'file_request': { - 'level': 'WARNING', - 'filters': ['require_debug_false'], - 'class': 'logging.FileHandler', - 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), - 'formatter': 'request' + "file": { + "level": "WARNING", + "filters": ["require_debug_false"], + "class": "logging.FileHandler", + "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), + "formatter": "simple", }, - 'console_verbose': { - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'verbose' + "file_request": { + "level": "WARNING", + "filters": ["require_debug_false"], + "class": "logging.FileHandler", + "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), + "formatter": "request", }, - 'file_verbose': { - 'level': 'WARNING', - 'filters': ['require_debug_false'], - 'class': 'logging.FileHandler', - 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), - 'formatter': 'verbose' + "console_verbose": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "verbose", }, - 'null': { - 'class': 'logging.NullHandler', + "file_verbose": { + "level": "WARNING", + "filters": ["require_debug_false"], + "class": "logging.FileHandler", + "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), + "formatter": "verbose", }, + "null": {"class": "logging.NullHandler",}, }, - 'loggers': { - '': { - 'handlers': ['console_verbose', 'file_verbose'], - 'level': 'DEBUG' if DEBUG else 'WARNING', + "loggers": { + "": { + "handlers": ["console_verbose", "file_verbose"], + "level": "DEBUG" if DEBUG else "WARNING", }, - 'django': { - 'handlers': ['console'], - 'level': 'DEBUG' if DEBUG else 'WARNING', - 'propagate': False, + "django": { + "handlers": ["console"], + "level": "DEBUG" if DEBUG else "WARNING", + "propagate": False, }, - 'django.request': { - 'handlers': ['file_request'], - 'level': 'DEBUG' if DEBUG else 'WARNING', - 'propagate': False, - }, - 'django.db.backends': { - 'handlers': ['null'], - 'propagate': False - }, - 'django.utils.autoreload': { - 'level': 'INFO', + "django.request": { + "handlers": ["file_request"], + "level": "DEBUG" if DEBUG else "WARNING", + "propagate": False, }, + "django.db.backends": {"handlers": ["null"], "propagate": False}, + "django.utils.autoreload": {"level": "INFO",}, }, } WEBPACK_LOADER = { - 'DEFAULT': { - 'CACHE': False, - 'BUNDLE_DIR_NAME': './', - 'STATS_FILE': os.path.join(STATIC_DIR, 'webpack-stats.json'), - 'POLL_INTERVAL': 0.1, - 'TIMEOUT': None, - 'IGNORE': ['.+\\.hot-update.js', '.+\\.map'] + "DEFAULT": { + "CACHE": False, + "BUNDLE_DIR_NAME": "./", + "STATS_FILE": os.path.join(STATIC_DIR, "webpack-stats.json"), + "POLL_INTERVAL": 0.1, + "TIMEOUT": None, + "IGNORE": [".+\\.hot-update.js", ".+\\.map"], } } -LOGIN_URL = '/admin/login/' -LOGIN_REDIRECT_URL = 'admin' +LOGIN_URL = "/admin/login/" +LOGIN_REDIRECT_URL = "admin" -SESSION_ENGINE = 'django.contrib.sessions.backends.cache' +SESSION_ENGINE = "django.contrib.sessions.backends.cache" CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' + "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, + "db_cache": { + "BACKEND": "django.core.cache.backends.db.DatabaseCache", + "LOCATION": "swh_web_cache", }, - 'db_cache': { - 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', - 'LOCATION': 'swh_web_cache', - } } JS_REVERSE_JS_MINIFY = False CORS_ORIGIN_ALLOW_ALL = True -CORS_URLS_REGEX = r'^/badge/.*$' +CORS_URLS_REGEX = r"^/badge/.*$" AUTHENTICATION_BACKENDS = [ - 'django.contrib.auth.backends.ModelBackend', - 'swh.web.auth.backends.OIDCAuthorizationCodePKCEBackend', + "django.contrib.auth.backends.ModelBackend", + "swh.web.auth.backends.OIDCAuthorizationCodePKCEBackend", ] diff --git a/swh/web/settings/development.py b/swh/web/settings/development.py index 15d12efb..8d152e3e 100644 --- a/swh/web/settings/development.py +++ b/swh/web/settings/development.py @@ -1,19 +1,19 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django development settings for swh-web. """ from django.core.cache import cache -from .common import * # noqa +from .common import * # noqa from .common import MIDDLEWARE -MIDDLEWARE += ['swh.web.common.middlewares.HtmlPrettifyMiddleware'] +MIDDLEWARE += ["swh.web.common.middlewares.HtmlPrettifyMiddleware"] AUTH_PASSWORD_VALIDATORS = [] # disable any pwd validation mechanism cache.clear() diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py index d96fc08f..4939387d 100644 --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -1,46 +1,50 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django production settings for swh-web. """ -from .common import * # noqa +from .common import * # noqa from .common import MIDDLEWARE, CACHES, WEBPACK_LOADER from .common import swh_web_config from .common import REST_FRAMEWORK # activate per-site caching -if 'GZip' in MIDDLEWARE[0]: - MIDDLEWARE.insert(1, 'django.middleware.cache.UpdateCacheMiddleware') +if "GZip" in MIDDLEWARE[0]: + MIDDLEWARE.insert(1, "django.middleware.cache.UpdateCacheMiddleware") else: - MIDDLEWARE.insert(0, 'django.middleware.cache.UpdateCacheMiddleware') - -MIDDLEWARE += ['swh.web.common.middlewares.HtmlMinifyMiddleware', - 'django.middleware.cache.FetchFromCacheMiddleware'] - -if swh_web_config.get('throttling', {}).get('cache_uri'): - CACHES.update({ - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': swh_web_config['throttling']['cache_uri'], + MIDDLEWARE.insert(0, "django.middleware.cache.UpdateCacheMiddleware") + +MIDDLEWARE += [ + "swh.web.common.middlewares.HtmlMinifyMiddleware", + "django.middleware.cache.FetchFromCacheMiddleware", +] + +if swh_web_config.get("throttling", {}).get("cache_uri"): + CACHES.update( + { + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": swh_web_config["throttling"]["cache_uri"], + } } - }) + ) # Setup support for proxy headers USE_X_FORWARDED_HOST = True -SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") # We're going through seven (or, in that case, 2) proxies thanks to Varnish -REST_FRAMEWORK['NUM_PROXIES'] = 2 +REST_FRAMEWORK["NUM_PROXIES"] = 2 DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': swh_web_config['production_db'], + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": swh_web_config["production_db"], } } -WEBPACK_LOADER['DEFAULT']['CACHE'] = True +WEBPACK_LOADER["DEFAULT"]["CACHE"] = True diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py index 6dfa08bb..e42235fc 100644 --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -1,115 +1,107 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django tests settings for swh-web. """ import os import sys from swh.web.config import get_config scope1_limiter_rate = 3 scope1_limiter_rate_post = 1 scope2_limiter_rate = 5 scope2_limiter_rate_post = 2 scope3_limiter_rate = 1 scope3_limiter_rate_post = 1 save_origin_rate_post = 10 swh_web_config = get_config() -swh_web_config.update({ - 'debug': False, - 'secret_key': 'test', - 'history_counters_url': '', - 'throttling': { - 'cache_uri': None, - 'scopes': { - 'swh_api': { - 'limiter_rate': { - 'default': '60/min' +swh_web_config.update( + { + "debug": False, + "secret_key": "test", + "history_counters_url": "", + "throttling": { + "cache_uri": None, + "scopes": { + "swh_api": { + "limiter_rate": {"default": "60/min"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_api_origin_search': { - 'limiter_rate': { - 'default': '100/min' + "swh_api_origin_search": { + "limiter_rate": {"default": "100/min"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_api_origin_visit_latest': { - 'limiter_rate': { - 'default': '6000/min' + "swh_api_origin_visit_latest": { + "limiter_rate": {"default": "6000/min"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_vault_cooking': { - 'limiter_rate': { - 'default': '120/h', - 'GET': '60/m' + "swh_vault_cooking": { + "limiter_rate": {"default": "120/h", "GET": "60/m"}, + "exempted_networks": ["127.0.0.0/8"], }, - 'exempted_networks': ['127.0.0.0/8'] - }, - 'swh_save_origin': { - 'limiter_rate': { - 'default': '120/h', - 'POST': '%s/h' % save_origin_rate_post, - } - }, - 'scope1': { - 'limiter_rate': { - 'default': '%s/min' % scope1_limiter_rate, - 'POST': '%s/min' % scope1_limiter_rate_post, - } - }, - 'scope2': { - 'limiter_rate': { - 'default': '%s/min' % scope2_limiter_rate, - 'POST': '%s/min' % scope2_limiter_rate_post - } - }, - 'scope3': { - 'limiter_rate': { - 'default': '%s/min' % scope3_limiter_rate, - 'POST': '%s/min' % scope3_limiter_rate_post + "swh_save_origin": { + "limiter_rate": { + "default": "120/h", + "POST": "%s/h" % save_origin_rate_post, + } }, - 'exempted_networks': ['127.0.0.0/8'] - } - } - }, - 'keycloak': { - 'server_url': 'http://localhost:8080/auth', - 'realm_name': 'SoftwareHeritage', - }, -}) + "scope1": { + "limiter_rate": { + "default": "%s/min" % scope1_limiter_rate, + "POST": "%s/min" % scope1_limiter_rate_post, + } + }, + "scope2": { + "limiter_rate": { + "default": "%s/min" % scope2_limiter_rate, + "POST": "%s/min" % scope2_limiter_rate_post, + } + }, + "scope3": { + "limiter_rate": { + "default": "%s/min" % scope3_limiter_rate, + "POST": "%s/min" % scope3_limiter_rate_post, + }, + "exempted_networks": ["127.0.0.0/8"], + }, + }, + }, + "keycloak": { + "server_url": "http://localhost:8080/auth", + "realm_name": "SoftwareHeritage", + }, + } +) -from .common import * # noqa -from .common import ALLOWED_HOSTS, LOGGING # noqa +from .common import * # noqa +from .common import ALLOWED_HOSTS, LOGGING # noqa DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': swh_web_config['test_db'], + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": swh_web_config["test_db"], } } # when not running unit tests, make the webapp fetch data from memory storages -if 'pytest' not in sys.argv[0] and 'PYTEST_XDIST_WORKER' not in os.environ: - swh_web_config.update({ - 'debug': True, - 'e2e_tests_mode': True - }) - from swh.web.tests.data import get_tests_data, override_storages # noqa +if "pytest" not in sys.argv[0] and "PYTEST_XDIST_WORKER" not in os.environ: + swh_web_config.update({"debug": True, "e2e_tests_mode": True}) + from swh.web.tests.data import get_tests_data, override_storages + test_data = get_tests_data() - override_storages(test_data['storage'], test_data['idx_storage'], - test_data['search']) + override_storages( + test_data["storage"], test_data["idx_storage"], test_data["search"] + ) else: - ALLOWED_HOSTS += ['testserver'] + ALLOWED_HOSTS += ["testserver"] # Silent DEBUG output when running unit tests - LOGGING['handlers']['console']['level'] = 'INFO' # type: ignore + LOGGING["handlers"]["console"]["level"] = "INFO" # type: ignore diff --git a/swh/web/tests/admin/test_origin_save.py b/swh/web/tests/admin/test_origin_save.py index d8555720..697565c8 100644 --- a/swh/web/tests/admin/test_origin_save.py +++ b/swh/web/tests/admin/test_origin_save.py @@ -1,223 +1,233 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from urllib.parse import unquote import pytest from django.contrib.auth import get_user_model from swh.web.common.models import ( - SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest + SaveAuthorizedOrigin, + SaveUnauthorizedOrigin, + SaveOriginRequest, ) from swh.web.common.origin_save import can_save_origin from swh.web.common.models import ( - SAVE_REQUEST_PENDING, SAVE_REQUEST_ACCEPTED, - SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_YET_SCHEDULED + SAVE_REQUEST_PENDING, + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_REJECTED, + SAVE_TASK_NOT_YET_SCHEDULED, ) from swh.web.common.utils import reverse -_user_name = 'swh-web-admin' -_user_mail = 'admin@swh-web.org' -_user_password = '..34~pounds~BEAUTY~march~63..' +_user_name = "swh-web-admin" +_user_mail = "admin@swh-web.org" +_user_password = "..34~pounds~BEAUTY~march~63.." -_authorized_origin_url = 'https://scm.ourproject.org/anonscm/' -_unauthorized_origin_url = 'https://www.softwareheritage.org/' +_authorized_origin_url = "https://scm.ourproject.org/anonscm/" +_unauthorized_origin_url = "https://www.softwareheritage.org/" pytestmark = pytest.mark.django_db @pytest.fixture(autouse=True) def populated_db(): User = get_user_model() user = User.objects.create_user(_user_name, _user_mail, _user_password) user.is_staff = True user.save() SaveAuthorizedOrigin.objects.create(url=_authorized_origin_url) SaveUnauthorizedOrigin.objects.create(url=_unauthorized_origin_url) def check_not_login(client, url): - login_url = reverse('login', query_params={'next': url}) + login_url = reverse("login", query_params={"next": url}) response = client.post(url) assert response.status_code == 302 assert unquote(response.url) == login_url def test_add_authorized_origin_url(client): - authorized_url = 'https://scm.adullact.net/anonscm/' + authorized_url = "https://scm.adullact.net/anonscm/" assert can_save_origin(authorized_url) == SAVE_REQUEST_PENDING - url = reverse('admin-origin-save-add-authorized-url', - url_args={'origin_url': authorized_url}) + url = reverse( + "admin-origin-save-add-authorized-url", url_args={"origin_url": authorized_url} + ) check_not_login(client, url) assert can_save_origin(authorized_url) == SAVE_REQUEST_PENDING client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(authorized_url) == SAVE_REQUEST_ACCEPTED def test_remove_authorized_origin_url(client): assert can_save_origin(_authorized_origin_url) == SAVE_REQUEST_ACCEPTED - url = reverse('admin-origin-save-remove-authorized-url', - url_args={'origin_url': _authorized_origin_url}) + url = reverse( + "admin-origin-save-remove-authorized-url", + url_args={"origin_url": _authorized_origin_url}, + ) check_not_login(client, url) assert can_save_origin(_authorized_origin_url) == SAVE_REQUEST_ACCEPTED client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(_authorized_origin_url) == SAVE_REQUEST_PENDING def test_add_unauthorized_origin_url(client): - unauthorized_url = 'https://www.yahoo./' + unauthorized_url = "https://www.yahoo./" assert can_save_origin(unauthorized_url) == SAVE_REQUEST_PENDING - url = reverse('admin-origin-save-add-unauthorized-url', - url_args={'origin_url': unauthorized_url}) + url = reverse( + "admin-origin-save-add-unauthorized-url", + url_args={"origin_url": unauthorized_url}, + ) check_not_login(client, url) assert can_save_origin(unauthorized_url) == SAVE_REQUEST_PENDING client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(unauthorized_url) == SAVE_REQUEST_REJECTED def test_remove_unauthorized_origin_url(client): assert can_save_origin(_unauthorized_origin_url) == SAVE_REQUEST_REJECTED - url = reverse('admin-origin-save-remove-unauthorized-url', - url_args={'origin_url': _unauthorized_origin_url}) + url = reverse( + "admin-origin-save-remove-unauthorized-url", + url_args={"origin_url": _unauthorized_origin_url}, + ) check_not_login(client, url) assert can_save_origin(_unauthorized_origin_url) == SAVE_REQUEST_REJECTED client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(_unauthorized_origin_url) == SAVE_REQUEST_PENDING def test_accept_pending_save_request(client, mocker): - mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') - visit_type = 'git' - origin_url = 'https://v2.pikacode.com/bthate/botlib.git' - save_request_url = reverse('api-1-save-origin', - url_args={'visit_type': visit_type, - 'origin_url': origin_url}) - response = client.post(save_request_url, data={}, - content_type='application/x-www-form-urlencoded') + mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") + visit_type = "git" + origin_url = "https://v2.pikacode.com/bthate/botlib.git" + save_request_url = reverse( + "api-1-save-origin", + url_args={"visit_type": visit_type, "origin_url": origin_url}, + ) + response = client.post( + save_request_url, data={}, content_type="application/x-www-form-urlencoded" + ) assert response.status_code == 200 - assert response.data['save_request_status'] == SAVE_REQUEST_PENDING + assert response.data["save_request_status"] == SAVE_REQUEST_PENDING - accept_request_url = reverse('admin-origin-save-request-accept', - url_args={'visit_type': visit_type, - 'origin_url': origin_url}) + accept_request_url = reverse( + "admin-origin-save-request-accept", + url_args={"visit_type": visit_type, "origin_url": origin_url}, + ) check_not_login(client, accept_request_url) tasks_data = [ { - 'priority': 'high', - 'policy': 'oneshot', - 'type': 'load-git', - 'arguments': { - 'kwargs': { - 'repo_url': origin_url - }, - 'args': [] - }, - 'status': 'next_run_not_scheduled', - 'id': 1, - } + "priority": "high", + "policy": "oneshot", + "type": "load-git", + "arguments": {"kwargs": {"repo_url": origin_url}, "args": []}, + "status": "next_run_not_scheduled", + "id": 1, + } ] mock_scheduler.create_tasks.return_value = tasks_data mock_scheduler.get_tasks.return_value = tasks_data client.login(username=_user_name, password=_user_password) response = client.post(accept_request_url) assert response.status_code == 200 response = client.get(save_request_url) assert response.status_code == 200 - assert response.data[0]['save_request_status'] == SAVE_REQUEST_ACCEPTED - assert response.data[0]['save_task_status'] == SAVE_TASK_NOT_YET_SCHEDULED + assert response.data[0]["save_request_status"] == SAVE_REQUEST_ACCEPTED + assert response.data[0]["save_task_status"] == SAVE_TASK_NOT_YET_SCHEDULED def test_reject_pending_save_request(client, mocker): - mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') - visit_type = 'git' - origin_url = 'https://wikipedia.com' - save_request_url = reverse('api-1-save-origin', - url_args={'visit_type': visit_type, - 'origin_url': origin_url}) - response = client.post(save_request_url, data={}, - content_type='application/x-www-form-urlencoded') + mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") + visit_type = "git" + origin_url = "https://wikipedia.com" + save_request_url = reverse( + "api-1-save-origin", + url_args={"visit_type": visit_type, "origin_url": origin_url}, + ) + response = client.post( + save_request_url, data={}, content_type="application/x-www-form-urlencoded" + ) assert response.status_code == 200 - assert response.data['save_request_status'] == SAVE_REQUEST_PENDING + assert response.data["save_request_status"] == SAVE_REQUEST_PENDING - reject_request_url = reverse('admin-origin-save-request-reject', - url_args={'visit_type': visit_type, - 'origin_url': origin_url}) + reject_request_url = reverse( + "admin-origin-save-request-reject", + url_args={"visit_type": visit_type, "origin_url": origin_url}, + ) check_not_login(client, reject_request_url) client.login(username=_user_name, password=_user_password) response = client.post(reject_request_url) assert response.status_code == 200 tasks_data = [ { - 'priority': 'high', - 'policy': 'oneshot', - 'type': 'load-git', - 'arguments': { - 'kwargs': { - 'repo_url': origin_url - }, - 'args': [] - }, - 'status': 'next_run_not_scheduled', - 'id': 1, + "priority": "high", + "policy": "oneshot", + "type": "load-git", + "arguments": {"kwargs": {"repo_url": origin_url}, "args": []}, + "status": "next_run_not_scheduled", + "id": 1, } ] mock_scheduler.create_tasks.return_value = tasks_data mock_scheduler.get_tasks.return_value = tasks_data response = client.get(save_request_url) assert response.status_code == 200 - assert response.data[0]['save_request_status'] == SAVE_REQUEST_REJECTED + assert response.data[0]["save_request_status"] == SAVE_REQUEST_REJECTED def test_remove_save_request(client): - sor = SaveOriginRequest.objects.create(visit_type='git', - origin_url='https://wikipedia.com', # noqa - status=SAVE_REQUEST_PENDING) + sor = SaveOriginRequest.objects.create( + visit_type="git", + origin_url="https://wikipedia.com", + status=SAVE_REQUEST_PENDING, + ) assert SaveOriginRequest.objects.count() == 1 - remove_request_url = reverse('admin-origin-save-request-remove', - url_args={'sor_id': sor.id}) + remove_request_url = reverse( + "admin-origin-save-request-remove", url_args={"sor_id": sor.id} + ) check_not_login(client, remove_request_url) client.login(username=_user_name, password=_user_password) response = client.post(remove_request_url) assert response.status_code == 200 assert SaveOriginRequest.objects.count() == 0 diff --git a/swh/web/tests/api/test_api_lookup.py b/swh/web/tests/api/test_api_lookup.py index 6c678078..2d7b47c7 100644 --- a/swh/web/tests/api/test_api_lookup.py +++ b/swh/web/tests/api/test_api_lookup.py @@ -1,115 +1,123 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.web.common.exc import NotFoundExc from swh.web.api.views import utils def test_genericapi_lookup_nothing_is_found(): - def test_generic_lookup_fn(sha1, another_unused_arg): - assert another_unused_arg == 'unused_arg' - assert sha1 == 'sha1' + assert another_unused_arg == "unused_arg" + assert sha1 == "sha1" return None - notfound_msg = 'This will be raised because None is returned.' + notfound_msg = "This will be raised because None is returned." with pytest.raises(NotFoundExc) as e: utils.api_lookup( - test_generic_lookup_fn, 'sha1', 'unused_arg', - notfound_msg=notfound_msg) + test_generic_lookup_fn, "sha1", "unused_arg", notfound_msg=notfound_msg + ) assert e.match(notfound_msg) def test_generic_api_map_are_enriched_and_transformed_to_list(): - def test_generic_lookup_fn_1(criteria0, param0, param1): - assert criteria0 == 'something' + assert criteria0 == "something" return map(lambda x: x + 1, [1, 2, 3]) actual_result = utils.api_lookup( - test_generic_lookup_fn_1, 'something', 'some param 0', - 'some param 1', - notfound_msg=('This is not the error message you are looking for. ' - 'Move along.'), - enrich_fn=lambda x, request: x * 2) + test_generic_lookup_fn_1, + "something", + "some param 0", + "some param 1", + notfound_msg=( + "This is not the error message you are looking for. " "Move along." + ), + enrich_fn=lambda x, request: x * 2, + ) assert actual_result == [4, 6, 8] def test_generic_api_list_are_enriched_too(): - def test_generic_lookup_fn_2(crit): - assert crit == 'something' - return ['a', 'b', 'c'] + assert crit == "something" + return ["a", "b", "c"] actual_result = utils.api_lookup( - test_generic_lookup_fn_2, 'something', - notfound_msg=('Not the error message you are looking for, it is. ' - 'Along, you move!'), - enrich_fn=lambda x, request: ''. join(['=', x, '='])) + test_generic_lookup_fn_2, + "something", + notfound_msg=( + "Not the error message you are looking for, it is. " "Along, you move!" + ), + enrich_fn=lambda x, request: "".join(["=", x, "="]), + ) - assert actual_result == ['=a=', '=b=', '=c='] + assert actual_result == ["=a=", "=b=", "=c="] def test_generic_api_generator_are_enriched_and_returned_as_list(): - def test_generic_lookup_fn_3(crit): - assert crit == 'crit' + assert crit == "crit" return (i for i in [4, 5, 6]) actual_result = utils.api_lookup( - test_generic_lookup_fn_3, 'crit', - notfound_msg='Move!', - enrich_fn=lambda x, request: x - 1) + test_generic_lookup_fn_3, + "crit", + notfound_msg="Move!", + enrich_fn=lambda x, request: x - 1, + ) assert actual_result == [3, 4, 5] def test_generic_api_simple_data_are_enriched_and_returned_too(): - def test_generic_lookup_fn_4(crit): - assert crit == '123' - return {'a': 10} + assert crit == "123" + return {"a": 10} def test_enrich_data(x, request): - x['a'] = x['a'] * 10 + x["a"] = x["a"] * 10 return x actual_result = utils.api_lookup( - test_generic_lookup_fn_4, '123', - notfound_msg='Nothing to do', - enrich_fn=test_enrich_data) + test_generic_lookup_fn_4, + "123", + notfound_msg="Nothing to do", + enrich_fn=test_enrich_data, + ) - assert actual_result == {'a': 100} + assert actual_result == {"a": 100} def test_api_lookup_not_found(): - notfound_msg = 'this is the error message raised as it is None' + notfound_msg = "this is the error message raised as it is None" with pytest.raises(NotFoundExc) as e: - utils.api_lookup( - lambda x: None, 'something', - notfound_msg=notfound_msg) + utils.api_lookup(lambda x: None, "something", notfound_msg=notfound_msg) assert e.match(notfound_msg) def test_api_lookup_with_result(): actual_result = utils.api_lookup( - lambda x: x + '!', 'something', - notfound_msg='this is the error which won\'t be used here') + lambda x: x + "!", + "something", + notfound_msg="this is the error which won't be used here", + ) - assert actual_result == 'something!' + assert actual_result == "something!" def test_api_lookup_with_result_as_map(): actual_result = utils.api_lookup( - lambda x: map(lambda y: y+1, x), [1, 2, 3], - notfound_msg='this is the error which won\'t be used here') + lambda x: map(lambda y: y + 1, x), + [1, 2, 3], + notfound_msg="this is the error which won't be used here", + ) assert actual_result == [2, 3, 4] diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py index a9768521..071c0b3a 100644 --- a/swh/web/tests/api/test_apidoc.py +++ b/swh/web/tests/api/test_apidoc.py @@ -1,499 +1,496 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import textwrap import pytest from rest_framework.response import Response from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api.apidoc import api_doc, _parse_httpdomain_doc from swh.web.api.apiurls import api_route from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc from swh.web.common.utils import reverse, prettify_html from swh.web.tests.django_asserts import assert_template_used _httpdomain_doc = """ .. http:get:: /api/1/revision/(sha1_git)/ Get information about a revision in the archive. Revisions are identified by **sha1** checksums, compatible with Git commit identifiers. See :func:`swh.model.identifiers.revision_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :json object author: information about the author of the revision :>json object committer: information about the committer of the revision :>json string committer_date: ISO representation of the commit date (in UTC) :>json string date: ISO representation of the revision date (in UTC) :>json string directory: the unique identifier that revision points to :>json string directory_url: link to :http:get:`/api/1/directory/(sha1_git)/[(path)/]` to get information about the directory associated to the revision :>json string id: the revision unique identifier :>json boolean merge: whether or not the revision corresponds to a merge commit :>json string message: the message associated to the revision :>json array parents: the parents of the revision, i.e. the previous revisions that head directly to it, each entry of that array contains an unique parent revision identifier but also a link to :http:get:`/api/1/revision/(sha1_git)/` to get more information about it :>json string type: the type of the revision :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Request:** .. parsed-literal:: :swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/` """ _exception_http_code = { BadInputExc: 400, ForbiddenExc: 403, NotFoundExc: 404, Exception: 500, StorageAPIError: 503, StorageDBError: 503, } def test_apidoc_nodoc_failure(): with pytest.raises(Exception): - @api_doc('/my/nodoc/url/') + + @api_doc("/my/nodoc/url/") def apidoc_nodoc_tester(request, arga=0, argb=0): return Response(arga + argb) -@api_route(r'/some/(?P[0-9]+)/(?P[0-9]+)/', - 'api-1-some-doc-route') -@api_doc('/some/doc/route/') +@api_route(r"/some/(?P[0-9]+)/(?P[0-9]+)/", "api-1-some-doc-route") +@api_doc("/some/doc/route/") def apidoc_route(request, myarg, myotherarg, akw=0): """ Sample doc """ - return {'result': int(myarg) + int(myotherarg) + akw} + return {"result": int(myarg) + int(myotherarg) + akw} def test_apidoc_route_doc(client): - url = reverse('api-1-some-doc-route-doc') - rv = client.get(url, HTTP_ACCEPT='text/html') + url = reverse("api-1-some-doc-route-doc") + rv = client.get(url, HTTP_ACCEPT="text/html") assert rv.status_code == 200, rv.content - assert_template_used(rv, 'api/apidoc.html') + assert_template_used(rv, "api/apidoc.html") def test_apidoc_route_fn(api_client): - url = reverse('api-1-some-doc-route', - url_args={'myarg': 1, 'myotherarg': 1}) + url = reverse("api-1-some-doc-route", url_args={"myarg": 1, "myotherarg": 1}) rv = api_client.get(url) assert rv.status_code == 200, rv.data -@api_route(r'/test/error/(?P.+)/', 'api-1-test-error') -@api_doc('/test/error/') +@api_route(r"/test/error/(?P.+)/", "api-1-test-error") +@api_doc("/test/error/") def apidoc_test_error_route(request, exc_name): """ Sample doc """ for e in _exception_http_code.keys(): if e.__name__ == exc_name: - raise e('Error') + raise e("Error") def test_apidoc_error(api_client): for exc, code in _exception_http_code.items(): - url = reverse('api-1-test-error', - url_args={'exc_name': exc.__name__}) + url = reverse("api-1-test-error", url_args={"exc_name": exc.__name__}) rv = api_client.get(url) assert rv.status_code == code, rv.data -@api_route(r'/some/full/(?P[0-9]+)/(?P[0-9]+)/', - 'api-1-some-complete-doc-route') -@api_doc('/some/complete/doc/route/') +@api_route( + r"/some/full/(?P[0-9]+)/(?P[0-9]+)/", + "api-1-some-complete-doc-route", +) +@api_doc("/some/complete/doc/route/") def apidoc_full_stack(request, myarg, myotherarg, akw=0): """ Sample doc """ - return {'result': int(myarg) + int(myotherarg) + akw} + return {"result": int(myarg) + int(myotherarg) + akw} def test_apidoc_full_stack_doc(client): - url = reverse('api-1-some-complete-doc-route-doc') - rv = client.get(url, HTTP_ACCEPT='text/html') + url = reverse("api-1-some-complete-doc-route-doc") + rv = client.get(url, HTTP_ACCEPT="text/html") assert rv.status_code == 200, rv.content - assert_template_used(rv, 'api/apidoc.html') + assert_template_used(rv, "api/apidoc.html") def test_apidoc_full_stack_fn(api_client): - url = reverse('api-1-some-complete-doc-route', - url_args={'myarg': 1, 'myotherarg': 1}) + url = reverse( + "api-1-some-complete-doc-route", url_args={"myarg": 1, "myotherarg": 1} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data -@api_route(r'/test/post/only/', 'api-1-test-post-only', - methods=['POST']) -@api_doc('/test/post/only/') +@api_route(r"/test/post/only/", "api-1-test-post-only", methods=["POST"]) +@api_doc("/test/post/only/") def apidoc_test_post_only(request, exc_name): """ Sample doc """ - return {'result': 'some data'} + return {"result": "some data"} def test_apidoc_post_only(client): # a dedicated view accepting GET requests should have # been created to display the HTML documentation - url = reverse('api-1-test-post-only-doc') - rv = client.get(url, HTTP_ACCEPT='text/html') + url = reverse("api-1-test-post-only-doc") + rv = client.get(url, HTTP_ACCEPT="text/html") assert rv.status_code == 200, rv.content - assert_template_used(rv, 'api/apidoc.html') + assert_template_used(rv, "api/apidoc.html") def test_api_doc_parse_httpdomain(): doc_data = { - 'description': '', - 'urls': [], - 'args': [], - 'params': [], - 'resheaders': [], - 'reqheaders': [], - 'input_type': '', - 'inputs': [], - 'return_type': '', - 'returns': [], - 'status_codes': [], - 'examples': [] + "description": "", + "urls": [], + "args": [], + "params": [], + "resheaders": [], + "reqheaders": [], + "input_type": "", + "inputs": [], + "return_type": "", + "returns": [], + "status_codes": [], + "examples": [], } _parse_httpdomain_doc(_httpdomain_doc, doc_data) - expected_urls = [{ - 'rule': '/api/1/revision/ **\\(sha1_git\\)** /', - 'methods': ['GET', 'HEAD', 'OPTIONS'] - }] + expected_urls = [ + { + "rule": "/api/1/revision/ **\\(sha1_git\\)** /", + "methods": ["GET", "HEAD", "OPTIONS"], + } + ] - assert 'urls' in doc_data - assert doc_data['urls'] == expected_urls + assert "urls" in doc_data + assert doc_data["urls"] == expected_urls - expected_description = ('Get information about a revision in the archive. ' - 'Revisions are identified by **sha1** checksums, ' - 'compatible with Git commit identifiers. See ' - '**swh.model.identifiers.revision_identifier** in ' - 'our data model module for details about how they ' - 'are computed.') + expected_description = ( + "Get information about a revision in the archive. " + "Revisions are identified by **sha1** checksums, " + "compatible with Git commit identifiers. See " + "**swh.model.identifiers.revision_identifier** in " + "our data model module for details about how they " + "are computed." + ) - assert 'description' in doc_data - assert doc_data['description'] == expected_description + assert "description" in doc_data + assert doc_data["description"] == expected_description - expected_args = [{ - 'name': 'sha1_git', - 'type': 'string', - 'doc': ('hexadecimal representation of the revision ' - '**sha1_git** identifier') - }] + expected_args = [ + { + "name": "sha1_git", + "type": "string", + "doc": ( + "hexadecimal representation of the revision " "**sha1_git** identifier" + ), + } + ] - assert 'args' in doc_data - assert doc_data['args'] == expected_args + assert "args" in doc_data + assert doc_data["args"] == expected_args expected_params = [] - assert 'params' in doc_data - assert doc_data['params'] == expected_params + assert "params" in doc_data + assert doc_data["params"] == expected_params - expected_reqheaders = [{ - 'doc': ('the requested response content type, either ' - '``application/json`` (default) or ``application/yaml``'), - 'name': 'Accept' - }] + expected_reqheaders = [ + { + "doc": ( + "the requested response content type, either " + "``application/json`` (default) or ``application/yaml``" + ), + "name": "Accept", + } + ] - assert 'reqheaders' in doc_data - assert doc_data['reqheaders'] == expected_reqheaders + assert "reqheaders" in doc_data + assert doc_data["reqheaders"] == expected_reqheaders - expected_resheaders = [{ - 'doc': 'this depends on **Accept** header of request', - 'name': 'Content-Type' - }] + expected_resheaders = [ + {"doc": "this depends on **Accept** header of request", "name": "Content-Type"} + ] - assert 'resheaders' in doc_data - assert doc_data['resheaders'] == expected_resheaders + assert "resheaders" in doc_data + assert doc_data["resheaders"] == expected_resheaders expected_statuscodes = [ - { - 'code': '200', - 'doc': 'no error' - }, - { - 'code': '400', - 'doc': 'an invalid **sha1_git** value has been provided' - }, - { - 'code': '404', - 'doc': 'requested revision can not be found in the archive' - } + {"code": "200", "doc": "no error"}, + {"code": "400", "doc": "an invalid **sha1_git** value has been provided"}, + {"code": "404", "doc": "requested revision can not be found in the archive"}, ] - assert 'status_codes' in doc_data - assert doc_data['status_codes'] == expected_statuscodes + assert "status_codes" in doc_data + assert doc_data["status_codes"] == expected_statuscodes - expected_input_type = 'object' + expected_input_type = "object" - assert 'input_type' in doc_data - assert doc_data['input_type'] == expected_input_type + assert "input_type" in doc_data + assert doc_data["input_type"] == expected_input_type expected_inputs = [ - { - 'name': 'n', - 'type': 'int', - 'doc': 'sample input integer' - }, - { - 'name': 's', - 'type': 'string', - 'doc': 'sample input string' - }, - { - 'name': 'a', - 'type': 'array', - 'doc': 'sample input array' - }, + {"name": "n", "type": "int", "doc": "sample input integer"}, + {"name": "s", "type": "string", "doc": "sample input string"}, + {"name": "a", "type": "array", "doc": "sample input array"}, ] - assert 'inputs' in doc_data - assert doc_data['inputs'] == expected_inputs + assert "inputs" in doc_data + assert doc_data["inputs"] == expected_inputs - expected_return_type = 'object' + expected_return_type = "object" - assert 'return_type' in doc_data - assert doc_data['return_type'] == expected_return_type + assert "return_type" in doc_data + assert doc_data["return_type"] == expected_return_type expected_returns = [ { - 'name': 'author', - 'type': 'object', - 'doc': 'information about the author of the revision' - }, - { - 'name': 'committer', - 'type': 'object', - 'doc': 'information about the committer of the revision' + "name": "author", + "type": "object", + "doc": "information about the author of the revision", }, { - 'name': 'committer_date', - 'type': 'string', - 'doc': 'ISO representation of the commit date (in UTC)' + "name": "committer", + "type": "object", + "doc": "information about the committer of the revision", }, { - 'name': 'date', - 'type': 'string', - 'doc': 'ISO representation of the revision date (in UTC)' + "name": "committer_date", + "type": "string", + "doc": "ISO representation of the commit date (in UTC)", }, { - 'name': 'directory', - 'type': 'string', - 'doc': 'the unique identifier that revision points to' + "name": "date", + "type": "string", + "doc": "ISO representation of the revision date (in UTC)", }, { - 'name': 'directory_url', - 'type': 'string', - 'doc': ('link to `/api/1/directory/ `_ ' - 'to get information about the directory associated to ' - 'the revision') + "name": "directory", + "type": "string", + "doc": "the unique identifier that revision points to", }, { - 'name': 'id', - 'type': 'string', - 'doc': 'the revision unique identifier' + "name": "directory_url", + "type": "string", + "doc": ( + "link to `/api/1/directory/ `_ " + "to get information about the directory associated to " + "the revision" + ), }, + {"name": "id", "type": "string", "doc": "the revision unique identifier"}, { - 'name': 'merge', - 'type': 'boolean', - 'doc': 'whether or not the revision corresponds to a merge commit' + "name": "merge", + "type": "boolean", + "doc": "whether or not the revision corresponds to a merge commit", }, { - 'name': 'message', - 'type': 'string', - 'doc': 'the message associated to the revision' + "name": "message", + "type": "string", + "doc": "the message associated to the revision", }, { - 'name': 'parents', - 'type': 'array', - 'doc': ('the parents of the revision, i.e. the previous revisions ' - 'that head directly to it, each entry of that array ' - 'contains an unique parent revision identifier but also a ' - 'link to `/api/1/revision/ `_ ' - 'to get more information about it') + "name": "parents", + "type": "array", + "doc": ( + "the parents of the revision, i.e. the previous revisions " + "that head directly to it, each entry of that array " + "contains an unique parent revision identifier but also a " + "link to `/api/1/revision/ `_ " + "to get more information about it" + ), }, - { - 'name': 'type', - 'type': 'string', - 'doc': 'the type of the revision' - } + {"name": "type", "type": "string", "doc": "the type of the revision"}, ] - assert 'returns' in doc_data - assert doc_data['returns'] == expected_returns + assert "returns" in doc_data + assert doc_data["returns"] == expected_returns - expected_examples = [ - '/api/1/revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/' - ] + expected_examples = ["/api/1/revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/"] - assert 'examples' in doc_data - assert doc_data['examples'] == expected_examples + assert "examples" in doc_data + assert doc_data["examples"] == expected_examples -@api_route(r'/post/endpoint/', 'api-1-post-endpoint', - methods=['POST']) -@api_doc('/post/endpoint/') +@api_route(r"/post/endpoint/", "api-1-post-endpoint", methods=["POST"]) +@api_doc("/post/endpoint/") def apidoc_test_post_endpoint(request): """ .. http:post:: /api/1/post/endpoint/ Endpoint documentation :json object : an object whose keys are input persistent identifiers and values objects with the following keys: * **known (bool)**: whether the object was found """ pass def test_apidoc_input_output_doc(client): - url = reverse('api-1-post-endpoint-doc') - rv = client.get(url, HTTP_ACCEPT='text/html') + url = reverse("api-1-post-endpoint-doc") + rv = client.get(url, HTTP_ACCEPT="text/html") assert rv.status_code == 200, rv.content - assert_template_used(rv, 'api/apidoc.html') - - input_html_doc = textwrap.indent(( - '
\n' - '
\n' - ' array\n' - '
\n' - '
\n' - '

\n' - ' Input array of pids\n' - '

\n' - '
\n' - '
\n' - ), ' '*7) - - output_html_doc = textwrap.indent(( - '
\n' - '
\n' - ' object\n' - '
\n' - '
\n' - '

\n' - ' an object containing the following keys:\n' - '

\n' - '
\n' - '
\n' - '
    \n' - '
  • \n' - '

    \n' - ' \n' - ' <swh_pid> (object)\n' - ' \n' - ' : an object whose keys are input persistent identifiers' - ' and values objects with the following keys:\n' - '

    \n' - '
    \n' - '
      \n' - '
    • \n' - '

      \n' - ' \n' - ' known (bool)\n' - ' \n' - ' : whether the object was found\n' - '

      \n' - '
      • \n' - '
    \n' - '
    \n' - '
    • \n' - '
\n' - '
\n' - '
\n' - '
\n' - '
\n' - ), ' '*7) + assert_template_used(rv, "api/apidoc.html") + + input_html_doc = textwrap.indent( + ( + '
\n' + '
\n' + " array\n" + "
\n" + '
\n' + "

\n" + " Input array of pids\n" + "

\n" + "
\n" + "
\n" + ), + " " * 7, + ) + + output_html_doc = textwrap.indent( + ( + '
\n' + '
\n' + " object\n" + "
\n" + '
\n' + "

\n" + " an object containing the following keys:\n" + "

\n" + '
\n' + "
\n" + "
    \n" + "
  • \n" + "

    \n" + " \n" + " <swh_pid> (object)\n" + " \n" + " : an object whose keys are input persistent identifiers" + " and values objects with the following keys:\n" + "

    \n" + "
    \n" + '
      \n' + "
    • \n" + "

      \n" + " \n" + " known (bool)\n" + " \n" + " : whether the object was found\n" + "

      \n" + "
      • \n" + "
    \n" + "
    \n" + "
    • \n" + "
\n" + "
\n" + "
\n" + "
\n" + "
\n" + ), + " " * 7, + ) html = prettify_html(rv.content) assert input_html_doc in html assert output_html_doc in html -@api_route(r'/endpoint/links/in/doc/', 'api-1-endpoint-links-in-doc') -@api_doc('/endpoint/links/in/doc/') +@api_route(r"/endpoint/links/in/doc/", "api-1-endpoint-links-in-doc") +@api_doc("/endpoint/links/in/doc/") def apidoc_test_endpoint_with_links_in_doc(request): """ .. http:get:: /api/1/post/endpoint/ Endpoint documentation with links to :http:get:`/api/1/content/[(hash_type):](hash)/`, :http:get:`/api/1/directory/(sha1_git)/[(path)/]` and `archive `_. """ pass def test_apidoc_with_links(client): - url = reverse('api-1-endpoint-links-in-doc') - rv = client.get(url, HTTP_ACCEPT='text/html') + url = reverse("api-1-endpoint-links-in-doc") + rv = client.get(url, HTTP_ACCEPT="text/html") assert rv.status_code == 200, rv.content - assert_template_used(rv, 'api/apidoc.html') + assert_template_used(rv, "api/apidoc.html") html = prettify_html(rv.content) - first_link = textwrap.indent(( - '\n' - ' /api/1/content/\n' - '' - ), ' '*9) - - second_link = textwrap.indent(( - '\n' - ' /api/1/directory/\n' - '' - ), ' '*9) - - third_link = textwrap.indent(( - '\n' - ' archive\n' - '' - ), ' '*9) + first_link = textwrap.indent( + ( + '\n' + " /api/1/content/\n" + "" + ), + " " * 9, + ) + + second_link = textwrap.indent( + ( + '\n' + " /api/1/directory/\n" + "" + ), + " " * 9, + ) + + third_link = textwrap.indent( + ( + '\n' + " archive\n" + "" + ), + " " * 9, + ) assert first_link in html assert second_link in html assert third_link in html diff --git a/swh/web/tests/api/test_apiresponse.py b/swh/web/tests/api/test_apiresponse.py index c82f97b0..2f610168 100644 --- a/swh/web/tests/api/test_apiresponse.py +++ b/swh/web/tests/api/test_apiresponse.py @@ -1,144 +1,140 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from swh.web.api.apiresponse import ( - compute_link_header, transform, make_api_response, - filter_by_fields + compute_link_header, + transform, + make_api_response, + filter_by_fields, ) def test_compute_link_header(): - next_link = '/api/endpoint/next' - prev_link = '/api/endpoint/prev' + next_link = "/api/endpoint/next" + prev_link = "/api/endpoint/prev" rv = { - 'headers': {'link-next': next_link, 'link-prev': prev_link}, - 'results': [1, 2, 3] + "headers": {"link-next": next_link, "link-prev": prev_link}, + "results": [1, 2, 3], } options = {} headers = compute_link_header(rv, options) assert headers == { - 'Link': (f'<{next_link}>; rel="next",' - f'<{prev_link}>; rel="previous"') + "Link": (f'<{next_link}>; rel="next",' f'<{prev_link}>; rel="previous"') } def test_compute_link_header_nothing_changed(): rv = {} options = {} headers = compute_link_header(rv, options) assert headers == {} def test_compute_link_header_nothing_changed_2(): - rv = {'headers': {}} + rv = {"headers": {}} options = {} headers = compute_link_header(rv, options) assert headers == {} def test_transform_only_return_results_1(): - rv = {'results': {'some-key': 'some-value'}} - assert transform(rv) == {'some-key': 'some-value'} + rv = {"results": {"some-key": "some-value"}} + assert transform(rv) == {"some-key": "some-value"} def test_transform_only_return_results_2(): - rv = {'headers': {'something': 'do changes'}, - 'results': {'some-key': 'some-value'}} - assert transform(rv) == {'some-key': 'some-value'} + rv = {"headers": {"something": "do changes"}, "results": {"some-key": "some-value"}} + assert transform(rv) == {"some-key": "some-value"} def test_transform_do_remove_headers(): - rv = {'headers': {'something': 'do changes'}, - 'some-key': 'some-value'} - assert transform(rv) == {'some-key': 'some-value'} + rv = {"headers": {"something": "do changes"}, "some-key": "some-value"} + assert transform(rv) == {"some-key": "some-value"} def test_transform_do_nothing(): - rv = {'some-key': 'some-value'} - assert transform(rv) == {'some-key': 'some-value'} + rv = {"some-key": "some-value"} + assert transform(rv) == {"some-key": "some-value"} def test_swh_multi_response_mimetype(mocker, api_request_factory): - mock_shorten_path = mocker.patch('swh.web.api.apiresponse.shorten_path') - mock_filter = mocker.patch('swh.web.api.apiresponse.filter_by_fields') - mock_json = mocker.patch('swh.web.api.apiresponse.json') + mock_shorten_path = mocker.patch("swh.web.api.apiresponse.shorten_path") + mock_filter = mocker.patch("swh.web.api.apiresponse.filter_by_fields") + mock_json = mocker.patch("swh.web.api.apiresponse.json") - data = { - 'data': [12, 34], - 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc' - } + data = {"data": [12, 34], "id": "adc83b19e793491b1c6ea0fd8b46cd9f32e592fc"} mock_filter.return_value = data - mock_shorten_path.return_value = 'my_short_path' + mock_shorten_path.return_value = "my_short_path" - accepted_response_formats = {'html': 'text/html', - 'yaml': 'application/yaml', - 'json': 'application/json'} + accepted_response_formats = { + "html": "text/html", + "yaml": "application/yaml", + "json": "application/json", + } for format in accepted_response_formats: - request = api_request_factory.get('/api/test/path/') + request = api_request_factory.get("/api/test/path/") mime_type = accepted_response_formats[format] - setattr(request, 'accepted_media_type', mime_type) + setattr(request, "accepted_media_type", mime_type) - if mime_type == 'text/html': + if mime_type == "text/html": expected_data = { - 'response_data': json.dumps(data), - 'headers_data': {}, - 'heading': 'my_short_path', - 'status_code': 200 + "response_data": json.dumps(data), + "headers_data": {}, + "heading": "my_short_path", + "status_code": 200, } mock_json.dumps.return_value = json.dumps(data) else: expected_data = data rv = make_api_response(request, data) mock_filter.assert_called_with(request, data) assert rv.status_code == 200, rv.data assert rv.data == expected_data - if mime_type == 'text/html': - assert rv.template_name == 'api/apidoc.html' + if mime_type == "text/html": + assert rv.template_name == "api/apidoc.html" def test_swh_filter_renderer_do_nothing(api_request_factory): - input_data = {'a': 'some-data'} + input_data = {"a": "some-data"} - request = api_request_factory.get('/api/test/path/', data={}) - setattr(request, 'query_params', request.GET) + request = api_request_factory.get("/api/test/path/", data={}) + setattr(request, "query_params", request.GET) actual_data = filter_by_fields(request, input_data) assert actual_data == input_data def test_swh_filter_renderer_do_filter(mocker, api_request_factory): - mock_ffk = mocker.patch('swh.web.api.apiresponse.utils.filter_field_keys') - mock_ffk.return_value = {'a': 'some-data'} + mock_ffk = mocker.patch("swh.web.api.apiresponse.utils.filter_field_keys") + mock_ffk.return_value = {"a": "some-data"} - request = api_request_factory.get('/api/test/path/', - data={'fields': 'a,c'}) - setattr(request, 'query_params', request.GET) + request = api_request_factory.get("/api/test/path/", data={"fields": "a,c"}) + setattr(request, "query_params", request.GET) - input_data = {'a': 'some-data', - 'b': 'some-other-data'} + input_data = {"a": "some-data", "b": "some-other-data"} actual_data = filter_by_fields(request, input_data) - assert actual_data == {'a': 'some-data'} + assert actual_data == {"a": "some-data"} - mock_ffk.assert_called_once_with(input_data, {'a', 'c'}) + mock_ffk.assert_called_once_with(input_data, {"a", "c"}) diff --git a/swh/web/tests/api/test_throttling.py b/swh/web/tests/api/test_throttling.py index 5d861125..c6f6495a 100644 --- a/swh/web/tests/api/test_throttling.py +++ b/swh/web/tests/api/test_throttling.py @@ -1,192 +1,194 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from django.conf.urls import url from django.contrib.auth.models import User from django.test.utils import override_settings from rest_framework.views import APIView from rest_framework.response import Response from rest_framework.decorators import api_view from swh.web.api.throttling import SwhWebRateThrottle, throttle_scope from swh.web.settings.tests import ( - scope1_limiter_rate, scope1_limiter_rate_post, - scope2_limiter_rate, scope2_limiter_rate_post, - scope3_limiter_rate, scope3_limiter_rate_post + scope1_limiter_rate, + scope1_limiter_rate_post, + scope2_limiter_rate, + scope2_limiter_rate_post, + scope3_limiter_rate, + scope3_limiter_rate_post, ) from swh.web.urls import urlpatterns class MockViewScope1(APIView): throttle_classes = (SwhWebRateThrottle,) - throttle_scope = 'scope1' + throttle_scope = "scope1" def get(self, request): - return Response('foo_get') + return Response("foo_get") def post(self, request): - return Response('foo_post') + return Response("foo_post") -@api_view(['GET', 'POST']) -@throttle_scope('scope2') +@api_view(["GET", "POST"]) +@throttle_scope("scope2") def mock_view_scope2(request): - if request.method == 'GET': - return Response('bar_get') - elif request.method == 'POST': - return Response('bar_post') + if request.method == "GET": + return Response("bar_get") + elif request.method == "POST": + return Response("bar_post") class MockViewScope3(APIView): throttle_classes = (SwhWebRateThrottle,) - throttle_scope = 'scope3' + throttle_scope = "scope3" def get(self, request): - return Response('foo_get') + return Response("foo_get") def post(self, request): - return Response('foo_post') + return Response("foo_post") -@api_view(['GET', 'POST']) -@throttle_scope('scope3') +@api_view(["GET", "POST"]) +@throttle_scope("scope3") def mock_view_scope3(request): - if request.method == 'GET': - return Response('bar_get') - elif request.method == 'POST': - return Response('bar_post') + if request.method == "GET": + return Response("bar_get") + elif request.method == "POST": + return Response("bar_post") urlpatterns += [ - url(r'^scope1_class$', MockViewScope1.as_view()), - url(r'^scope2_func$', mock_view_scope2), - url(r'^scope3_class$', MockViewScope3.as_view()), - url(r'^scope3_func$', mock_view_scope3) + url(r"^scope1_class$", MockViewScope1.as_view()), + url(r"^scope2_func$", mock_view_scope2), + url(r"^scope3_class$", MockViewScope3.as_view()), + url(r"^scope3_func$", mock_view_scope3), ] -def check_response(response, status_code, - limit=None, remaining=None): +def check_response(response, status_code, limit=None, remaining=None): assert response.status_code == status_code if limit is not None: - assert response['X-RateLimit-Limit'] == str(limit) + assert response["X-RateLimit-Limit"] == str(limit) else: - assert 'X-RateLimit-Limit' not in response + assert "X-RateLimit-Limit" not in response if remaining is not None: - assert response['X-RateLimit-Remaining'] == str(remaining) + assert response["X-RateLimit-Remaining"] == str(remaining) else: - assert 'X-RateLimit-Remaining' not in response + assert "X-RateLimit-Remaining" not in response @override_settings(ROOT_URLCONF=__name__) def test_scope1_requests_are_throttled(api_client): """ Ensure request rate is limited in scope1 """ for i in range(scope1_limiter_rate): - response = api_client.get('/scope1_class') - check_response(response, 200, scope1_limiter_rate, - scope1_limiter_rate - i - 1) + response = api_client.get("/scope1_class") + check_response(response, 200, scope1_limiter_rate, scope1_limiter_rate - i - 1) - response = api_client.get('/scope1_class') + response = api_client.get("/scope1_class") check_response(response, 429, scope1_limiter_rate, 0) for i in range(scope1_limiter_rate_post): - response = api_client.post('/scope1_class') - check_response(response, 200, scope1_limiter_rate_post, - scope1_limiter_rate_post - i - 1) + response = api_client.post("/scope1_class") + check_response( + response, 200, scope1_limiter_rate_post, scope1_limiter_rate_post - i - 1 + ) - response = api_client.post('/scope1_class') + response = api_client.post("/scope1_class") check_response(response, 429, scope1_limiter_rate_post, 0) @override_settings(ROOT_URLCONF=__name__) def test_scope2_requests_are_throttled(api_client): """ Ensure request rate is limited in scope2 """ for i in range(scope2_limiter_rate): - response = api_client.get('/scope2_func') - check_response(response, 200, scope2_limiter_rate, - scope2_limiter_rate - i - 1) + response = api_client.get("/scope2_func") + check_response(response, 200, scope2_limiter_rate, scope2_limiter_rate - i - 1) - response = api_client.get('/scope2_func') + response = api_client.get("/scope2_func") check_response(response, 429, scope2_limiter_rate, 0) for i in range(scope2_limiter_rate_post): - response = api_client.post('/scope2_func') - check_response(response, 200, scope2_limiter_rate_post, - scope2_limiter_rate_post - i - 1) + response = api_client.post("/scope2_func") + check_response( + response, 200, scope2_limiter_rate_post, scope2_limiter_rate_post - i - 1 + ) - response = api_client.post('/scope2_func') + response = api_client.post("/scope2_func") check_response(response, 429, scope2_limiter_rate_post, 0) @override_settings(ROOT_URLCONF=__name__) def test_scope3_requests_are_throttled_exempted(api_client): """ Ensure request rate is not limited in scope3 as requests coming from localhost are exempted from rate limit. """ - for _ in range(scope3_limiter_rate+1): - response = api_client.get('/scope3_class') + for _ in range(scope3_limiter_rate + 1): + response = api_client.get("/scope3_class") check_response(response, 200) - for _ in range(scope3_limiter_rate_post+1): - response = api_client.post('/scope3_class') + for _ in range(scope3_limiter_rate_post + 1): + response = api_client.post("/scope3_class") check_response(response, 200) - for _ in range(scope3_limiter_rate+1): - response = api_client.get('/scope3_func') + for _ in range(scope3_limiter_rate + 1): + response = api_client.get("/scope3_func") check_response(response, 200) - for _ in range(scope3_limiter_rate_post+1): - response = api_client.post('/scope3_func') + for _ in range(scope3_limiter_rate_post + 1): + response = api_client.post("/scope3_func") check_response(response, 200) @override_settings(ROOT_URLCONF=__name__) @pytest.mark.django_db def test_staff_users_are_not_rate_limited(api_client): staff_user = User.objects.create_user( - username='johndoe', password='', is_staff=True) + username="johndoe", password="", is_staff=True + ) api_client.force_login(staff_user) - for _ in range(scope2_limiter_rate+1): - response = api_client.get('/scope2_func') + for _ in range(scope2_limiter_rate + 1): + response = api_client.get("/scope2_func") check_response(response, 200) - for _ in range(scope2_limiter_rate_post+1): - response = api_client.post('/scope2_func') + for _ in range(scope2_limiter_rate_post + 1): + response = api_client.post("/scope2_func") check_response(response, 200) @override_settings(ROOT_URLCONF=__name__) @pytest.mark.django_db def test_non_staff_users_are_rate_limited(api_client): - user = User.objects.create_user( - username='johndoe', password='', is_staff=False) + user = User.objects.create_user(username="johndoe", password="", is_staff=False) api_client.force_login(user) for i in range(scope2_limiter_rate): - response = api_client.get('/scope2_func') - check_response(response, 200, scope2_limiter_rate, - scope2_limiter_rate - i - 1) + response = api_client.get("/scope2_func") + check_response(response, 200, scope2_limiter_rate, scope2_limiter_rate - i - 1) - response = api_client.get('/scope2_func') + response = api_client.get("/scope2_func") check_response(response, 429, scope2_limiter_rate, 0) for i in range(scope2_limiter_rate_post): - response = api_client.post('/scope2_func') - check_response(response, 200, scope2_limiter_rate_post, - scope2_limiter_rate_post - i - 1) + response = api_client.post("/scope2_func") + check_response( + response, 200, scope2_limiter_rate_post, scope2_limiter_rate_post - i - 1 + ) - response = api_client.post('/scope2_func') + response = api_client.post("/scope2_func") check_response(response, 429, scope2_limiter_rate_post, 0) diff --git a/swh/web/tests/api/test_utils.py b/swh/web/tests/api/test_utils.py index 5a906029..5033dde7 100644 --- a/swh/web/tests/api/test_utils.py +++ b/swh/web/tests/api/test_utils.py @@ -1,601 +1,603 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import random from hypothesis import given from swh.model.hashutil import DEFAULT_ALGORITHMS from swh.web.api import utils from swh.web.common.origin_visits import get_origin_visits from swh.web.common.utils import reverse, resolve_branch_alias from swh.web.tests.strategies import ( - release, directory, content, revision, snapshot, origin + release, + directory, + content, + revision, + snapshot, + origin, ) url_map = [ { - 'rule': '/other/', - 'methods': set(['GET', 'POST', 'HEAD']), - 'endpoint': 'foo' + "rule": "/other/", + "methods": set(["GET", "POST", "HEAD"]), + "endpoint": "foo", }, { - 'rule': '/some/old/url/', - 'methods': set(['GET', 'POST']), - 'endpoint': 'blablafn' + "rule": "/some/old/url/", + "methods": set(["GET", "POST"]), + "endpoint": "blablafn", }, { - 'rule': '/other/old/url/', - 'methods': set(['GET', 'HEAD']), - 'endpoint': 'bar' + "rule": "/other/old/url/", + "methods": set(["GET", "HEAD"]), + "endpoint": "bar", }, - { - 'rule': '/other', - 'methods': set([]), - 'endpoint': None - }, - { - 'rule': '/other2', - 'methods': set([]), - 'endpoint': None - } + {"rule": "/other", "methods": set([]), "endpoint": None}, + {"rule": "/other2", "methods": set([]), "endpoint": None}, ] def test_filter_field_keys_dict_unknown_keys(): actual_res = utils.filter_field_keys( - {'directory': 1, 'file': 2, 'link': 3}, - {'directory1', 'file2'}) + {"directory": 1, "file": 2, "link": 3}, {"directory1", "file2"} + ) assert actual_res == {} def test_filter_field_keys_dict(): actual_res = utils.filter_field_keys( - {'directory': 1, 'file': 2, 'link': 3}, - {'directory', 'link'}) + {"directory": 1, "file": 2, "link": 3}, {"directory", "link"} + ) - assert actual_res == {'directory': 1, 'link': 3} + assert actual_res == {"directory": 1, "link": 3} def test_filter_field_keys_list_unknown_keys(): actual_res = utils.filter_field_keys( - [{'directory': 1, 'file': 2, 'link': 3}, - {'1': 1, '2': 2, 'link': 3}], {'d'}) + [{"directory": 1, "file": 2, "link": 3}, {"1": 1, "2": 2, "link": 3}], {"d"} + ) assert actual_res == [{}, {}] def test_filter_field_keys_map(): actual_res = utils.filter_field_keys( - map(lambda x: {'i': x['i']+1, 'j': x['j']}, - [{'i': 1, 'j': None}, - {'i': 2, 'j': None}, - {'i': 3, 'j': None}]), {'i'}) + map( + lambda x: {"i": x["i"] + 1, "j": x["j"]}, + [{"i": 1, "j": None}, {"i": 2, "j": None}, {"i": 3, "j": None}], + ), + {"i"}, + ) - assert list(actual_res) == [{'i': 2}, {'i': 3}, {'i': 4}] + assert list(actual_res) == [{"i": 2}, {"i": 3}, {"i": 4}] def test_filter_field_keys_list(): actual_res = utils.filter_field_keys( - [{'directory': 1, 'file': 2, 'link': 3}, - {'dir': 1, 'fil': 2, 'lin': 3}], - {'directory', 'dir'}) + [{"directory": 1, "file": 2, "link": 3}, {"dir": 1, "fil": 2, "lin": 3}], + {"directory", "dir"}, + ) - assert actual_res == [{'directory': 1}, {'dir': 1}] + assert actual_res == [{"directory": 1}, {"dir": 1}] def test_filter_field_keys_other(): input_set = {1, 2} - actual_res = utils.filter_field_keys(input_set, {'a', '1'}) + actual_res = utils.filter_field_keys(input_set, {"a", "1"}) assert actual_res == input_set def test_person_to_string(): - assert utils.person_to_string({'name': 'raboof', - 'email': 'foo@bar'}) == 'raboof ' + assert ( + utils.person_to_string({"name": "raboof", "email": "foo@bar"}) + == "raboof " + ) def test_enrich_release_empty(): actual_release = utils.enrich_release({}) assert actual_release == {} @given(release()) -def test_enrich_release_content_target(api_request_factory, - archive_data, release): +def test_enrich_release_content_target(api_request_factory, archive_data, release): release_data = archive_data.release_get(release) - release_data['target_type'] = 'content' + release_data["target_type"] = "content" - url = reverse('api-1-release', url_args={'sha1_git': release}) + url = reverse("api-1-release", url_args={"sha1_git": release}) request = api_request_factory.get(url) actual_release = utils.enrich_release(release_data, request) - release_data['target_url'] = reverse( - 'api-1-content', - url_args={'q': f'sha1_git:{release_data["target"]}'}, - request=request) + release_data["target_url"] = reverse( + "api-1-content", + url_args={"q": f'sha1_git:{release_data["target"]}'}, + request=request, + ) assert actual_release == release_data @given(release()) -def test_enrich_release_directory_target(api_request_factory, - archive_data, release): +def test_enrich_release_directory_target(api_request_factory, archive_data, release): release_data = archive_data.release_get(release) - release_data['target_type'] = 'directory' + release_data["target_type"] = "directory" - url = reverse('api-1-release', url_args={'sha1_git': release}) + url = reverse("api-1-release", url_args={"sha1_git": release}) request = api_request_factory.get(url) actual_release = utils.enrich_release(release_data, request) - release_data['target_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': release_data['target']}, - request=request) + release_data["target_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": release_data["target"]}, + request=request, + ) assert actual_release == release_data @given(release()) -def test_enrich_release_revision_target(api_request_factory, - archive_data, release): +def test_enrich_release_revision_target(api_request_factory, archive_data, release): release_data = archive_data.release_get(release) - release_data['target_type'] = 'revision' + release_data["target_type"] = "revision" - url = reverse('api-1-release', url_args={'sha1_git': release}) + url = reverse("api-1-release", url_args={"sha1_git": release}) request = api_request_factory.get(url) actual_release = utils.enrich_release(release_data, request) - release_data['target_url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': release_data['target']}, - request=request) + release_data["target_url"] = reverse( + "api-1-revision", url_args={"sha1_git": release_data["target"]}, request=request + ) assert actual_release == release_data @given(release()) -def test_enrich_release_release_target(api_request_factory, - archive_data, release): +def test_enrich_release_release_target(api_request_factory, archive_data, release): release_data = archive_data.release_get(release) - release_data['target_type'] = 'release' + release_data["target_type"] = "release" - url = reverse('api-1-release', url_args={'sha1_git': release}) + url = reverse("api-1-release", url_args={"sha1_git": release}) request = api_request_factory.get(url) actual_release = utils.enrich_release(release_data, request) - release_data['target_url'] = reverse( - 'api-1-release', - url_args={'sha1_git': release_data['target']}, - request=request) + release_data["target_url"] = reverse( + "api-1-release", url_args={"sha1_git": release_data["target"]}, request=request + ) assert actual_release == release_data def test_enrich_directory_no_type(): - assert utils.enrich_directory({'id': 'dir-id'}) == {'id': 'dir-id'} + assert utils.enrich_directory({"id": "dir-id"}) == {"id": "dir-id"} @given(directory()) -def test_enrich_directory_with_type(api_request_factory, - archive_data, directory): +def test_enrich_directory_with_type(api_request_factory, archive_data, directory): dir_content = archive_data.directory_ls(directory) dir_entry = random.choice(dir_content) - url = reverse('api-1-directory', url_args={'sha1_git': directory}) + url = reverse("api-1-directory", url_args={"sha1_git": directory}) request = api_request_factory.get(url) actual_directory = utils.enrich_directory(dir_entry, request) - if dir_entry['type'] == 'file': - dir_entry['target_url'] = reverse( - 'api-1-content', - url_args={'q': f'sha1_git:{dir_entry["target"]}'}, - request=request) - - elif dir_entry['type'] == 'dir': - dir_entry['target_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': dir_entry['target']}, - request=request) - - elif dir_entry['type'] == 'rev': - dir_entry['target_url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': dir_entry['target']}, - request=request) + if dir_entry["type"] == "file": + dir_entry["target_url"] = reverse( + "api-1-content", + url_args={"q": f'sha1_git:{dir_entry["target"]}'}, + request=request, + ) + + elif dir_entry["type"] == "dir": + dir_entry["target_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": dir_entry["target"]}, + request=request, + ) + + elif dir_entry["type"] == "rev": + dir_entry["target_url"] = reverse( + "api-1-revision", + url_args={"sha1_git": dir_entry["target"]}, + request=request, + ) assert actual_directory == dir_entry def test_enrich_content_without_hashes(): - assert utils.enrich_content({'id': '123'}) == {'id': '123'} + assert utils.enrich_content({"id": "123"}) == {"id": "123"} @given(content()) def test_enrich_content_with_hashes(api_request_factory, content): for algo in DEFAULT_ALGORITHMS: content_data = dict(content) - query_string = '%s:%s' % (algo, content_data[algo]) + query_string = "%s:%s" % (algo, content_data[algo]) - url = reverse('api-1-content', url_args={'q': query_string}) + url = reverse("api-1-content", url_args={"q": query_string}) request = api_request_factory.get(url) - enriched_content = utils.enrich_content(content_data, - query_string=query_string, - request=request) + enriched_content = utils.enrich_content( + content_data, query_string=query_string, request=request + ) - content_data['data_url'] = reverse('api-1-content-raw', - url_args={'q': query_string}, - request=request) + content_data["data_url"] = reverse( + "api-1-content-raw", url_args={"q": query_string}, request=request + ) - content_data['filetype_url'] = reverse('api-1-content-filetype', - url_args={'q': query_string}, - request=request) + content_data["filetype_url"] = reverse( + "api-1-content-filetype", url_args={"q": query_string}, request=request + ) - content_data['language_url'] = reverse('api-1-content-language', - url_args={'q': query_string}, - request=request) + content_data["language_url"] = reverse( + "api-1-content-language", url_args={"q": query_string}, request=request + ) - content_data['license_url'] = reverse('api-1-content-license', - url_args={'q': query_string}, - request=request) + content_data["license_url"] = reverse( + "api-1-content-license", url_args={"q": query_string}, request=request + ) assert enriched_content == content_data @given(content()) -def test_enrich_content_with_hashes_and_top_level_url(api_request_factory, - content): +def test_enrich_content_with_hashes_and_top_level_url(api_request_factory, content): for algo in DEFAULT_ALGORITHMS: content_data = dict(content) - query_string = '%s:%s' % (algo, content_data[algo]) + query_string = "%s:%s" % (algo, content_data[algo]) - url = reverse('api-1-content', url_args={'q': query_string}) + url = reverse("api-1-content", url_args={"q": query_string}) request = api_request_factory.get(url) - enriched_content = utils.enrich_content(content_data, - query_string=query_string, - top_url=True, - request=request) + enriched_content = utils.enrich_content( + content_data, query_string=query_string, top_url=True, request=request + ) - content_data['content_url'] = reverse('api-1-content', - url_args={'q': query_string}, - request=request) + content_data["content_url"] = reverse( + "api-1-content", url_args={"q": query_string}, request=request + ) - content_data['data_url'] = reverse('api-1-content-raw', - url_args={'q': query_string}, - request=request) + content_data["data_url"] = reverse( + "api-1-content-raw", url_args={"q": query_string}, request=request + ) - content_data['filetype_url'] = reverse('api-1-content-filetype', - url_args={'q': query_string}, - request=request) + content_data["filetype_url"] = reverse( + "api-1-content-filetype", url_args={"q": query_string}, request=request + ) - content_data['language_url'] = reverse('api-1-content-language', - url_args={'q': query_string}, - request=request) + content_data["language_url"] = reverse( + "api-1-content-language", url_args={"q": query_string}, request=request + ) - content_data['license_url'] = reverse('api-1-content-license', - url_args={'q': query_string}, - request=request) + content_data["license_url"] = reverse( + "api-1-content-license", url_args={"q": query_string}, request=request + ) assert enriched_content == content_data @given(revision()) -def test_enrich_revision_without_children_or_parent(api_request_factory, - archive_data, revision): +def test_enrich_revision_without_children_or_parent( + api_request_factory, archive_data, revision +): revision_data = archive_data.revision_get(revision) - del revision_data['parents'] + del revision_data["parents"] - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) request = api_request_factory.get(url) actual_revision = utils.enrich_revision(revision_data, request) - revision_data['url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': revision}, - request=request) + revision_data["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision}, request=request + ) - revision_data['history_url'] = reverse( - 'api-1-revision-log', - url_args={'sha1_git': revision}, - request=request) + revision_data["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision}, request=request + ) - revision_data['directory_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': revision_data['directory']}, - request=request) + revision_data["directory_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": revision_data["directory"]}, + request=request, + ) assert actual_revision == revision_data @given(revision(), revision(), revision()) -def test_enrich_revision_with_children_and_parent_no_dir(api_request_factory, - archive_data, - revision, - parent_revision, - child_revision): +def test_enrich_revision_with_children_and_parent_no_dir( + api_request_factory, archive_data, revision, parent_revision, child_revision +): revision_data = archive_data.revision_get(revision) - del revision_data['directory'] - revision_data['parents'].append(parent_revision) - revision_data['children'] = child_revision + del revision_data["directory"] + revision_data["parents"].append(parent_revision) + revision_data["children"] = child_revision - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) request = api_request_factory.get(url) actual_revision = utils.enrich_revision(revision_data, request) - revision_data['url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': revision}, - request=request) - - revision_data['history_url'] = reverse( - 'api-1-revision-log', - url_args={'sha1_git': revision}, - request=request) - - revision_data['parents'] = [ - {'id': p['id'], 'url': reverse('api-1-revision', - url_args={'sha1_git': p['id']}, - request=request)} - for p in revision_data['parents'] + revision_data["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision}, request=request + ) + + revision_data["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision}, request=request + ) + + revision_data["parents"] = [ + { + "id": p["id"], + "url": reverse( + "api-1-revision", url_args={"sha1_git": p["id"]}, request=request + ), + } + for p in revision_data["parents"] ] - revision_data['children_urls'] = [ - reverse('api-1-revision', - url_args={'sha1_git': child_revision}, - request=request) + revision_data["children_urls"] = [ + reverse( + "api-1-revision", url_args={"sha1_git": child_revision}, request=request + ) ] assert actual_revision == revision_data @given(revision(), revision(), revision()) -def test_enrich_revision_no_context(api_request_factory, - revision, - parent_revision, - child_revision): +def test_enrich_revision_no_context( + api_request_factory, revision, parent_revision, child_revision +): revision_data = { - 'id': revision, - 'parents': [parent_revision], - 'children': [child_revision] + "id": revision, + "parents": [parent_revision], + "children": [child_revision], } - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) request = api_request_factory.get(url) actual_revision = utils.enrich_revision(revision_data, request) - revision_data['url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': revision}, - request=request) - - revision_data['history_url'] = reverse( - 'api-1-revision-log', - url_args={'sha1_git': revision}, - request=request) - - revision_data['parents'] = [{ - 'id': parent_revision, - 'url': reverse('api-1-revision', - url_args={'sha1_git': parent_revision}, - request=request) - }] - - revision_data['children_urls'] = [ - reverse('api-1-revision', - url_args={'sha1_git': child_revision}, - request=request) + revision_data["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision}, request=request + ) + + revision_data["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision}, request=request + ) + + revision_data["parents"] = [ + { + "id": parent_revision, + "url": reverse( + "api-1-revision", + url_args={"sha1_git": parent_revision}, + request=request, + ), + } + ] + + revision_data["children_urls"] = [ + reverse( + "api-1-revision", url_args={"sha1_git": child_revision}, request=request + ) ] assert actual_revision == revision_data @given(revision(), revision(), revision()) -def test_enrich_revision_with_no_message(api_request_factory, - archive_data, - revision, - parent_revision, - child_revision): +def test_enrich_revision_with_no_message( + api_request_factory, archive_data, revision, parent_revision, child_revision +): revision_data = archive_data.revision_get(revision) - revision_data['message'] = None - revision_data['parents'].append(parent_revision) - revision_data['children'] = child_revision + revision_data["message"] = None + revision_data["parents"].append(parent_revision) + revision_data["children"] = child_revision - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) request = api_request_factory.get(url) actual_revision = utils.enrich_revision(revision_data, request) - revision_data['url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': revision}, - request=request) - - revision_data['directory_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': revision_data['directory']}, - request=request) - - revision_data['history_url'] = reverse( - 'api-1-revision-log', - url_args={'sha1_git': revision}, - request=request) - - revision_data['parents'] = [ - {'id': p['id'], 'url': reverse('api-1-revision', - url_args={'sha1_git': p['id']}, - request=request)} - for p in revision_data['parents'] + revision_data["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision}, request=request + ) + + revision_data["directory_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": revision_data["directory"]}, + request=request, + ) + + revision_data["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision}, request=request + ) + + revision_data["parents"] = [ + { + "id": p["id"], + "url": reverse( + "api-1-revision", url_args={"sha1_git": p["id"]}, request=request + ), + } + for p in revision_data["parents"] ] - revision_data['children_urls'] = [ - reverse('api-1-revision', - url_args={'sha1_git': child_revision}, - request=request) + revision_data["children_urls"] = [ + reverse( + "api-1-revision", url_args={"sha1_git": child_revision}, request=request + ) ] assert actual_revision == revision_data @given(revision(), revision(), revision()) -def test_enrich_revision_with_invalid_message(api_request_factory, - archive_data, - revision, - parent_revision, - child_revision): +def test_enrich_revision_with_invalid_message( + api_request_factory, archive_data, revision, parent_revision, child_revision +): revision_data = archive_data.revision_get(revision) - revision_data['message'] = None - revision_data['message_decoding_failed'] = True, - revision_data['parents'].append(parent_revision) - revision_data['children'] = child_revision + revision_data["message"] = None + revision_data["message_decoding_failed"] = (True,) + revision_data["parents"].append(parent_revision) + revision_data["children"] = child_revision - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) request = api_request_factory.get(url) actual_revision = utils.enrich_revision(revision_data, request) - revision_data['url'] = reverse( - 'api-1-revision', - url_args={'sha1_git': revision}, - request=request) - - revision_data['message_url'] = reverse( - 'api-1-revision-raw-message', - url_args={'sha1_git': revision}, - request=request) - - revision_data['directory_url'] = reverse( - 'api-1-directory', - url_args={'sha1_git': revision_data['directory']}, - request=request) - - revision_data['history_url'] = reverse( - 'api-1-revision-log', - url_args={'sha1_git': revision}, - request=request) - - revision_data['parents'] = [ - {'id': p['id'], 'url': reverse('api-1-revision', - url_args={'sha1_git': p['id']}, - request=request)} - for p in revision_data['parents'] + revision_data["url"] = reverse( + "api-1-revision", url_args={"sha1_git": revision}, request=request + ) + + revision_data["message_url"] = reverse( + "api-1-revision-raw-message", url_args={"sha1_git": revision}, request=request + ) + + revision_data["directory_url"] = reverse( + "api-1-directory", + url_args={"sha1_git": revision_data["directory"]}, + request=request, + ) + + revision_data["history_url"] = reverse( + "api-1-revision-log", url_args={"sha1_git": revision}, request=request + ) + + revision_data["parents"] = [ + { + "id": p["id"], + "url": reverse( + "api-1-revision", url_args={"sha1_git": p["id"]}, request=request + ), + } + for p in revision_data["parents"] ] - revision_data['children_urls'] = [ - reverse('api-1-revision', - url_args={'sha1_git': child_revision}, - request=request) + revision_data["children_urls"] = [ + reverse( + "api-1-revision", url_args={"sha1_git": child_revision}, request=request + ) ] assert actual_revision == revision_data @given(snapshot()) def test_enrich_snapshot(api_request_factory, archive_data, snapshot): snapshot_data = archive_data.snapshot_get(snapshot) - url = reverse('api-1-snapshot', url_args={'snapshot_id': snapshot}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": snapshot}) request = api_request_factory.get(url) actual_snapshot = utils.enrich_snapshot(snapshot_data, request) - for _, b in snapshot_data['branches'].items(): - if b['target_type'] in ('directory', 'revision', 'release'): - b['target_url'] = reverse(f'api-1-{b["target_type"]}', - url_args={'sha1_git': b['target']}, - request=request) - elif b['target_type'] == 'content': - b['target_url'] = reverse( - 'api-1-content', - url_args={'q': f'sha1_git:{b["target"]}'}, - request=request) - - for _, b in snapshot_data['branches'].items(): - if b['target_type'] == 'alias': + for _, b in snapshot_data["branches"].items(): + if b["target_type"] in ("directory", "revision", "release"): + b["target_url"] = reverse( + f'api-1-{b["target_type"]}', + url_args={"sha1_git": b["target"]}, + request=request, + ) + elif b["target_type"] == "content": + b["target_url"] = reverse( + "api-1-content", + url_args={"q": f'sha1_git:{b["target"]}'}, + request=request, + ) + + for _, b in snapshot_data["branches"].items(): + if b["target_type"] == "alias": target = resolve_branch_alias(snapshot_data, b) - b['target_url'] = target['target_url'] + b["target_url"] = target["target_url"] assert actual_snapshot == snapshot_data @given(origin()) def test_enrich_origin(api_request_factory, archive_data, origin): - url = reverse('api-1-origin', url_args={'origin_url': origin['url']}) + url = reverse("api-1-origin", url_args={"origin_url": origin["url"]}) request = api_request_factory.get(url) - origin_data = {'url': origin['url']} + origin_data = {"url": origin["url"]} actual_origin = utils.enrich_origin(origin_data, request) - origin_data['origin_visits_url'] = reverse( - 'api-1-origin-visits', - url_args={'origin_url': origin['url']}, - request=request) + origin_data["origin_visits_url"] = reverse( + "api-1-origin-visits", url_args={"origin_url": origin["url"]}, request=request + ) assert actual_origin == origin_data @given(origin()) def test_enrich_origin_visit(api_request_factory, archive_data, origin): origin_visit = random.choice(get_origin_visits(origin)) - url = reverse('api-1-origin-visit', - url_args={'origin_url': origin['url'], - 'visit_id': origin_visit['visit']}) + url = reverse( + "api-1-origin-visit", + url_args={"origin_url": origin["url"], "visit_id": origin_visit["visit"]}, + ) request = api_request_factory.get(url) actual_origin_visit = utils.enrich_origin_visit( - origin_visit, with_origin_link=True, - with_origin_visit_link=True, request=request) - - origin_visit['origin_url'] = reverse( - 'api-1-origin', - url_args={'origin_url': origin['url']}, - request=request) - - origin_visit['origin_visit_url'] = reverse( - 'api-1-origin-visit', - url_args={'origin_url': origin['url'], - 'visit_id': origin_visit['visit']}, - request=request) - - origin_visit['snapshot_url'] = reverse( - 'api-1-snapshot', - url_args={'snapshot_id': origin_visit['snapshot']}, - request=request) + origin_visit, + with_origin_link=True, + with_origin_visit_link=True, + request=request, + ) + + origin_visit["origin_url"] = reverse( + "api-1-origin", url_args={"origin_url": origin["url"]}, request=request + ) + + origin_visit["origin_visit_url"] = reverse( + "api-1-origin-visit", + url_args={"origin_url": origin["url"], "visit_id": origin_visit["visit"]}, + request=request, + ) + + origin_visit["snapshot_url"] = reverse( + "api-1-snapshot", + url_args={"snapshot_id": origin_visit["snapshot"]}, + request=request, + ) assert actual_origin_visit == origin_visit diff --git a/swh/web/tests/api/views/test_content.py b/swh/web/tests/api/views/test_content.py index 95ddec97..dc831770 100644 --- a/swh/web/tests/api/views/test_content.py +++ b/swh/web/tests/api/views/test_content.py @@ -1,387 +1,399 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from hypothesis import given from swh.web.common.utils import reverse from swh.web.tests.data import random_content from swh.web.tests.strategies import content, contents_with_ctags from swh.web.tests.conftest import ctags_json_missing, fossology_missing @given(content()) def test_api_content_filetype(api_client, indexer_data, content): - indexer_data.content_add_mimetype(content['sha1']) - url = reverse('api-1-content-filetype', - url_args={'q': 'sha1_git:%s' % content['sha1_git']}) + indexer_data.content_add_mimetype(content["sha1"]) + url = reverse( + "api-1-content-filetype", url_args={"q": "sha1_git:%s" % content["sha1_git"]} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - content_url = reverse('api-1-content', - url_args={'q': 'sha1:%s' % content['sha1']}, - request=rv.wsgi_request) - expected_data = indexer_data.content_get_mimetype(content['sha1']) - expected_data['content_url'] = content_url + assert rv["Content-Type"] == "application/json" + content_url = reverse( + "api-1-content", + url_args={"q": "sha1:%s" % content["sha1"]}, + request=rv.wsgi_request, + ) + expected_data = indexer_data.content_get_mimetype(content["sha1"]) + expected_data["content_url"] = content_url assert rv.data == expected_data def test_api_content_filetype_sha_not_found(api_client): unknown_content_ = random_content() - url = reverse('api-1-content-filetype', - url_args={'q': 'sha1:%s' % unknown_content_['sha1']}) + url = reverse( + "api-1-content-filetype", url_args={"q": "sha1:%s" % unknown_content_["sha1"]} + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'No filetype information found for content ' - 'sha1:%s.' % unknown_content_['sha1'] + "exception": "NotFoundExc", + "reason": "No filetype information found for content " + "sha1:%s." % unknown_content_["sha1"], } @pytest.mark.skip # Language indexer is disabled @given(content()) def test_api_content_language(api_client, indexer_data, content): - indexer_data.content_add_language(content['sha1']) - url = reverse('api-1-content-language', - url_args={'q': 'sha1_git:%s' % content['sha1_git']}) + indexer_data.content_add_language(content["sha1"]) + url = reverse( + "api-1-content-language", url_args={"q": "sha1_git:%s" % content["sha1_git"]} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - content_url = reverse('api-1-content', - url_args={'q': 'sha1:%s' % content['sha1']}, - request=rv.wsgi_request) - expected_data = indexer_data.content_get_language(content['sha1']) - expected_data['content_url'] = content_url + assert rv["Content-Type"] == "application/json" + content_url = reverse( + "api-1-content", + url_args={"q": "sha1:%s" % content["sha1"]}, + request=rv.wsgi_request, + ) + expected_data = indexer_data.content_get_language(content["sha1"]) + expected_data["content_url"] = content_url assert rv.data == expected_data def test_api_content_language_sha_not_found(api_client): unknown_content_ = random_content() - url = reverse('api-1-content-language', - url_args={'q': 'sha1:%s' % unknown_content_['sha1']}) + url = reverse( + "api-1-content-language", url_args={"q": "sha1:%s" % unknown_content_["sha1"]} + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'No language information found for content ' - 'sha1:%s.' % unknown_content_['sha1'] + "exception": "NotFoundExc", + "reason": "No language information found for content " + "sha1:%s." % unknown_content_["sha1"], } @pytest.mark.skip # Language indexer is disabled -@pytest.mark.skipif(ctags_json_missing, - reason="requires ctags with json output support") +@pytest.mark.skipif( + ctags_json_missing, reason="requires ctags with json output support" +) @given(contents_with_ctags()) def test_api_content_symbol(api_client, indexer_data, contents_with_ctags): expected_data = {} - for content_sha1 in contents_with_ctags['sha1s']: + for content_sha1 in contents_with_ctags["sha1s"]: indexer_data.content_add_ctags(content_sha1) for ctag in indexer_data.content_get_ctags(content_sha1): - if ctag['name'] == contents_with_ctags['symbol_name']: + if ctag["name"] == contents_with_ctags["symbol_name"]: expected_data[content_sha1] = ctag break - url = reverse('api-1-content-symbol', - url_args={'q': contents_with_ctags['symbol_name']}, - query_params={'per_page': 100}) + url = reverse( + "api-1-content-symbol", + url_args={"q": contents_with_ctags["symbol_name"]}, + query_params={"per_page": 100}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" for entry in rv.data: - content_sha1 = entry['sha1'] + content_sha1 = entry["sha1"] expected_entry = expected_data[content_sha1] - for key, view_name in (('content_url', 'api-1-content'), - ('data_url', 'api-1-content-raw'), - ('license_url', 'api-1-content-license'), - ('language_url', 'api-1-content-language'), - ('filetype_url', 'api-1-content-filetype')): + for key, view_name in ( + ("content_url", "api-1-content"), + ("data_url", "api-1-content-raw"), + ("license_url", "api-1-content-license"), + ("language_url", "api-1-content-language"), + ("filetype_url", "api-1-content-filetype"), + ): expected_entry[key] = reverse( - view_name, url_args={'q': 'sha1:%s' % content_sha1}, - request=rv.wsgi_request) - expected_entry['sha1'] = content_sha1 - del expected_entry['id'] + view_name, + url_args={"q": "sha1:%s" % content_sha1}, + request=rv.wsgi_request, + ) + expected_entry["sha1"] = content_sha1 + del expected_entry["id"] assert entry == expected_entry - assert 'Link' not in rv + assert "Link" not in rv - url = reverse('api-1-content-symbol', - url_args={'q': contents_with_ctags['symbol_name']}, - query_params={'per_page': 2}) + url = reverse( + "api-1-content-symbol", + url_args={"q": contents_with_ctags["symbol_name"]}, + query_params={"per_page": 2}, + ) rv = api_client.get(url) - next_url = reverse('api-1-content-symbol', - url_args={'q': contents_with_ctags['symbol_name']}, - query_params={'last_sha1': rv.data[1]['sha1'], - 'per_page': 2}, - request=rv.wsgi_request), - assert rv['Link'] == '<%s>; rel="next"' % next_url + next_url = ( + reverse( + "api-1-content-symbol", + url_args={"q": contents_with_ctags["symbol_name"]}, + query_params={"last_sha1": rv.data[1]["sha1"], "per_page": 2}, + request=rv.wsgi_request, + ), + ) + assert rv["Link"] == '<%s>; rel="next"' % next_url def test_api_content_symbol_not_found(api_client): - url = reverse('api-1-content-symbol', url_args={'q': 'bar'}) + url = reverse("api-1-content-symbol", url_args={"q": "bar"}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'No indexed raw content match expression \'bar\'.' + "exception": "NotFoundExc", + "reason": "No indexed raw content match expression 'bar'.", } - assert 'Link' not in rv + assert "Link" not in rv -@pytest.mark.skipif(ctags_json_missing, - reason="requires ctags with json output support") +@pytest.mark.skipif( + ctags_json_missing, reason="requires ctags with json output support" +) @given(content()) def test_api_content_ctags(api_client, indexer_data, content): - indexer_data.content_add_ctags(content['sha1']) - url = reverse('api-1-content-ctags', - url_args={'q': 'sha1_git:%s' % content['sha1_git']}) + indexer_data.content_add_ctags(content["sha1"]) + url = reverse( + "api-1-content-ctags", url_args={"q": "sha1_git:%s" % content["sha1_git"]} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - content_url = reverse('api-1-content', - url_args={'q': 'sha1:%s' % content['sha1']}, - request=rv.wsgi_request) - expected_data = list(indexer_data.content_get_ctags(content['sha1'])) + assert rv["Content-Type"] == "application/json" + content_url = reverse( + "api-1-content", + url_args={"q": "sha1:%s" % content["sha1"]}, + request=rv.wsgi_request, + ) + expected_data = list(indexer_data.content_get_ctags(content["sha1"])) for e in expected_data: - e['content_url'] = content_url + e["content_url"] = content_url assert rv.data == expected_data -@pytest.mark.skipif(fossology_missing, - reason="requires fossology-nomossa installed") +@pytest.mark.skipif(fossology_missing, reason="requires fossology-nomossa installed") @given(content()) def test_api_content_license(api_client, indexer_data, content): - indexer_data.content_add_license(content['sha1']) - url = reverse('api-1-content-license', - url_args={'q': 'sha1_git:%s' % content['sha1_git']}) + indexer_data.content_add_license(content["sha1"]) + url = reverse( + "api-1-content-license", url_args={"q": "sha1_git:%s" % content["sha1_git"]} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - content_url = reverse('api-1-content', - url_args={'q': 'sha1:%s' % content['sha1']}, - request=rv.wsgi_request) - expected_data = indexer_data.content_get_license(content['sha1']) - expected_data['content_url'] = content_url + assert rv["Content-Type"] == "application/json" + content_url = reverse( + "api-1-content", + url_args={"q": "sha1:%s" % content["sha1"]}, + request=rv.wsgi_request, + ) + expected_data = indexer_data.content_get_license(content["sha1"]) + expected_data["content_url"] = content_url assert rv.data == expected_data def test_api_content_license_sha_not_found(api_client): unknown_content_ = random_content() - url = reverse('api-1-content-license', - url_args={'q': 'sha1:%s' % unknown_content_['sha1']}) + url = reverse( + "api-1-content-license", url_args={"q": "sha1:%s" % unknown_content_["sha1"]} + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'No license information found for content ' - 'sha1:%s.' % unknown_content_['sha1'] + "exception": "NotFoundExc", + "reason": "No license information found for content " + "sha1:%s." % unknown_content_["sha1"], } @given(content()) def test_api_content_metadata(api_client, archive_data, content): - url = reverse('api-1-content', {'q': 'sha1:%s' % content['sha1']}) + url = reverse("api-1-content", {"q": "sha1:%s" % content["sha1"]}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - expected_data = archive_data.content_get_metadata(content['sha1']) - for key, view_name in (('data_url', 'api-1-content-raw'), - ('license_url', 'api-1-content-license'), - ('language_url', 'api-1-content-language'), - ('filetype_url', 'api-1-content-filetype')): + assert rv["Content-Type"] == "application/json" + expected_data = archive_data.content_get_metadata(content["sha1"]) + for key, view_name in ( + ("data_url", "api-1-content-raw"), + ("license_url", "api-1-content-license"), + ("language_url", "api-1-content-language"), + ("filetype_url", "api-1-content-filetype"), + ): expected_data[key] = reverse( - view_name, url_args={'q': 'sha1:%s' % content['sha1']}, - request=rv.wsgi_request) + view_name, + url_args={"q": "sha1:%s" % content["sha1"]}, + request=rv.wsgi_request, + ) assert rv.data == expected_data def test_api_content_not_found_as_json(api_client): unknown_content_ = random_content() - url = reverse('api-1-content', - url_args={'q': 'sha1:%s' % unknown_content_['sha1']}) + url = reverse("api-1-content", url_args={"q": "sha1:%s" % unknown_content_["sha1"]}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Content with sha1 checksum equals to %s not found!' - % unknown_content_['sha1'] + "exception": "NotFoundExc", + "reason": "Content with sha1 checksum equals to %s not found!" + % unknown_content_["sha1"], } def test_api_content_not_found_as_yaml(api_client): unknown_content_ = random_content() - url = reverse('api-1-content', - url_args={'q': 'sha256:%s' % unknown_content_['sha256']}) - rv = api_client.get(url, HTTP_ACCEPT='application/yaml') + url = reverse( + "api-1-content", url_args={"q": "sha256:%s" % unknown_content_["sha256"]} + ) + rv = api_client.get(url, HTTP_ACCEPT="application/yaml") assert rv.status_code == 404, rv.data - assert 'application/yaml' in rv['Content-Type'] + assert "application/yaml" in rv["Content-Type"] assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Content with sha256 checksum equals to %s not found!' % - unknown_content_['sha256'] + "exception": "NotFoundExc", + "reason": "Content with sha256 checksum equals to %s not found!" + % unknown_content_["sha256"], } def test_api_content_raw_ko_not_found(api_client): unknown_content_ = random_content() - url = reverse('api-1-content-raw', - url_args={'q': 'sha1:%s' % unknown_content_['sha1']}) + url = reverse( + "api-1-content-raw", url_args={"q": "sha1:%s" % unknown_content_["sha1"]} + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Content with sha1 checksum equals to %s not found!' % - unknown_content_['sha1'] + "exception": "NotFoundExc", + "reason": "Content with sha1 checksum equals to %s not found!" + % unknown_content_["sha1"], } @given(content()) def test_api_content_raw_text(api_client, archive_data, content): - url = reverse('api-1-content-raw', - url_args={'q': 'sha1:%s' % content['sha1']}) + url = reverse("api-1-content-raw", url_args={"q": "sha1:%s" % content["sha1"]}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/octet-stream' - assert rv['Content-disposition'] == \ - 'attachment; filename=content_sha1_%s_raw' % content['sha1'] - assert rv['Content-Type'] == 'application/octet-stream' - expected_data = archive_data.content_get(content['sha1']) - assert rv.content == expected_data['data'] + assert rv["Content-Type"] == "application/octet-stream" + assert ( + rv["Content-disposition"] + == "attachment; filename=content_sha1_%s_raw" % content["sha1"] + ) + assert rv["Content-Type"] == "application/octet-stream" + expected_data = archive_data.content_get(content["sha1"]) + assert rv.content == expected_data["data"] @given(content()) def test_api_content_raw_text_with_filename(api_client, archive_data, content): - url = reverse('api-1-content-raw', - url_args={'q': 'sha1:%s' % content['sha1']}, - query_params={'filename': 'filename.txt'}) + url = reverse( + "api-1-content-raw", + url_args={"q": "sha1:%s" % content["sha1"]}, + query_params={"filename": "filename.txt"}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/octet-stream' - assert rv['Content-disposition'] == \ - 'attachment; filename=filename.txt' - assert rv['Content-Type'] == 'application/octet-stream' - expected_data = archive_data.content_get(content['sha1']) - assert rv.content == expected_data['data'] + assert rv["Content-Type"] == "application/octet-stream" + assert rv["Content-disposition"] == "attachment; filename=filename.txt" + assert rv["Content-Type"] == "application/octet-stream" + expected_data = archive_data.content_get(content["sha1"]) + assert rv.content == expected_data["data"] @given(content()) def test_api_check_content_known(api_client, content): - url = reverse('api-1-content-known', - url_args={'q': content['sha1']}) + url = reverse("api-1-content-known", url_args={"q": content["sha1"]}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'search_res': [ - { - 'found': True, - 'sha1': content['sha1'] - } - ], - 'search_stats': {'nbfiles': 1, 'pct': 100.0} + "search_res": [{"found": True, "sha1": content["sha1"]}], + "search_stats": {"nbfiles": 1, "pct": 100.0}, } @given(content()) def test_api_check_content_known_as_yaml(api_client, content): - url = reverse('api-1-content-known', - url_args={'q': content['sha1']}) - rv = api_client.get(url, HTTP_ACCEPT='application/yaml') + url = reverse("api-1-content-known", url_args={"q": content["sha1"]}) + rv = api_client.get(url, HTTP_ACCEPT="application/yaml") assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/yaml' + assert rv["Content-Type"] == "application/yaml" assert rv.data == { - 'search_res': [ - { - 'found': True, - 'sha1': content['sha1'] - } - ], - 'search_stats': {'nbfiles': 1, 'pct': 100.0} + "search_res": [{"found": True, "sha1": content["sha1"]}], + "search_stats": {"nbfiles": 1, "pct": 100.0}, } @given(content()) def test_api_check_content_known_post_as_yaml(api_client, content): - url = reverse('api-1-content-known') - rv = api_client.post(url, data={'q': content['sha1']}, - HTTP_ACCEPT='application/yaml') + url = reverse("api-1-content-known") + rv = api_client.post( + url, data={"q": content["sha1"]}, HTTP_ACCEPT="application/yaml" + ) assert rv.status_code == 200, rv.data - assert 'application/yaml' in rv['Content-Type'] + assert "application/yaml" in rv["Content-Type"] assert rv.data == { - 'search_res': [ - { - 'found': True, - 'sha1': content['sha1'] - } - ], - 'search_stats': {'nbfiles': 1, 'pct': 100.0} + "search_res": [{"found": True, "sha1": content["sha1"]}], + "search_stats": {"nbfiles": 1, "pct": 100.0}, } def test_api_check_content_known_not_found(api_client): unknown_content_ = random_content() - url = reverse('api-1-content-known', - url_args={'q': unknown_content_['sha1']}) + url = reverse("api-1-content-known", url_args={"q": unknown_content_["sha1"]}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'search_res': [ - { - 'found': False, - 'sha1': unknown_content_['sha1'] - } - ], - 'search_stats': {'nbfiles': 1, 'pct': 0.0} + "search_res": [{"found": False, "sha1": unknown_content_["sha1"]}], + "search_stats": {"nbfiles": 1, "pct": 0.0}, } @given(content()) def test_api_content_uppercase(api_client, content): - url = reverse('api-1-content-uppercase-checksum', - url_args={'q': content['sha1'].upper()}) + url = reverse( + "api-1-content-uppercase-checksum", url_args={"q": content["sha1"].upper()} + ) rv = api_client.get(url) assert rv.status_code == 302, rv.data - redirect_url = reverse('api-1-content', - url_args={'q': content['sha1']}) + redirect_url = reverse("api-1-content", url_args={"q": content["sha1"]}) - assert rv['location'] == redirect_url + assert rv["location"] == redirect_url diff --git a/swh/web/tests/api/views/test_directory.py b/swh/web/tests/api/views/test_directory.py index 0f1d184e..ab4bc54e 100644 --- a/swh/web/tests/api/views/test_directory.py +++ b/swh/web/tests/api/views/test_directory.py @@ -1,90 +1,91 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import random from hypothesis import given from swh.web.api.utils import enrich_directory from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 from swh.web.tests.strategies import directory @given(directory()) def test_api_directory(api_client, archive_data, directory): - url = reverse('api-1-directory', url_args={'sha1_git': directory}) + url = reverse("api-1-directory", url_args={"sha1_git": directory}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" dir_content = list(archive_data.directory_ls(directory)) - expected_data = list(map(enrich_directory, - dir_content, - [rv.wsgi_request] * len(dir_content))) + expected_data = list( + map(enrich_directory, dir_content, [rv.wsgi_request] * len(dir_content)) + ) assert rv.data == expected_data def test_api_directory_not_found(api_client): unknown_directory_ = random_sha1() - url = reverse('api-1-directory', - url_args={'sha1_git': unknown_directory_}) + url = reverse("api-1-directory", url_args={"sha1_git": unknown_directory_}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Directory with sha1_git %s not found' % unknown_directory_ + "exception": "NotFoundExc", + "reason": "Directory with sha1_git %s not found" % unknown_directory_, } @given(directory()) def test_api_directory_with_path_found(api_client, archive_data, directory): directory_content = archive_data.directory_ls(directory) path = random.choice(directory_content) - url = reverse('api-1-directory', - url_args={'sha1_git': directory, 'path': path['name']}) + url = reverse( + "api-1-directory", url_args={"sha1_git": directory, "path": path["name"]} + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == enrich_directory(path, rv.wsgi_request) @given(directory()) def test_api_directory_with_path_not_found(api_client, directory): - path = 'some/path/to/nonexistent/dir/' - url = reverse('api-1-directory', - url_args={'sha1_git': directory, 'path': path}) + path = "some/path/to/nonexistent/dir/" + url = reverse("api-1-directory", url_args={"sha1_git": directory, "path": path}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': ('Directory entry with path %s from %s not found' % - (path, directory)) + "exception": "NotFoundExc", + "reason": ( + "Directory entry with path %s from %s not found" % (path, directory) + ), } @given(directory()) def test_api_directory_uppercase(api_client, directory): - url = reverse('api-1-directory-uppercase-checksum', - url_args={'sha1_git': directory.upper()}) + url = reverse( + "api-1-directory-uppercase-checksum", url_args={"sha1_git": directory.upper()} + ) resp = api_client.get(url) assert resp.status_code == 302 - redirect_url = reverse('api-1-directory', url_args={'sha1_git': directory}) + redirect_url = reverse("api-1-directory", url_args={"sha1_git": directory}) - assert resp['location'] == redirect_url + assert resp["location"] == redirect_url diff --git a/swh/web/tests/api/views/test_identifiers.py b/swh/web/tests/api/views/test_identifiers.py index cc200d2b..e297ed9e 100644 --- a/swh/web/tests/api/views/test_identifiers.py +++ b/swh/web/tests/api/views/test_identifiers.py @@ -1,161 +1,188 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import given -from swh.model.identifiers import ( - CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT -) +from swh.model.identifiers import CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 from swh.web.tests.strategies import ( - content, directory, origin, release, revision, snapshot, - unknown_content, unknown_directory, unknown_release, - unknown_revision, unknown_snapshot + content, + directory, + origin, + release, + revision, + snapshot, + unknown_content, + unknown_directory, + unknown_release, + unknown_revision, + unknown_snapshot, ) @given(origin(), content(), directory(), release(), revision(), snapshot()) -def test_swh_id_resolve_success(api_client, origin, content, directory, - release, revision, snapshot): +def test_swh_id_resolve_success( + api_client, origin, content, directory, release, revision, snapshot +): for obj_type_short, obj_type, obj_id in ( - ('cnt', CONTENT, content['sha1_git']), - ('dir', DIRECTORY, directory), - ('rel', RELEASE, release), - ('rev', REVISION, revision), - ('snp', SNAPSHOT, snapshot)): + ("cnt", CONTENT, content["sha1_git"]), + ("dir", DIRECTORY, directory), + ("rel", RELEASE, release), + ("rev", REVISION, revision), + ("snp", SNAPSHOT, snapshot), + ): - swh_id = 'swh:1:%s:%s;origin=%s' % (obj_type_short, obj_id, - origin['url']) - url = reverse('api-1-resolve-swh-pid', url_args={'swh_id': swh_id}) + swh_id = "swh:1:%s:%s;origin=%s" % (obj_type_short, obj_id, origin["url"]) + url = reverse("api-1-resolve-swh-pid", url_args={"swh_id": swh_id}) resp = api_client.get(url) if obj_type == CONTENT: - url_args = {'query_string': 'sha1_git:%s' % obj_id} + url_args = {"query_string": "sha1_git:%s" % obj_id} elif obj_type == SNAPSHOT: - url_args = {'snapshot_id': obj_id} + url_args = {"snapshot_id": obj_id} else: - url_args = {'sha1_git': obj_id} + url_args = {"sha1_git": obj_id} - browse_rev_url = reverse('browse-%s' % obj_type, - url_args=url_args, - query_params={'origin': origin['url']}, - request=resp.wsgi_request) + browse_rev_url = reverse( + "browse-%s" % obj_type, + url_args=url_args, + query_params={"origin": origin["url"]}, + request=resp.wsgi_request, + ) expected_result = { - 'browse_url': browse_rev_url, - 'metadata': {'origin': origin['url']}, - 'namespace': 'swh', - 'object_id': obj_id, - 'object_type': obj_type, - 'scheme_version': 1 + "browse_url": browse_rev_url, + "metadata": {"origin": origin["url"]}, + "namespace": "swh", + "object_id": obj_id, + "object_type": obj_type, + "scheme_version": 1, } assert resp.status_code == 200, resp.data assert resp.data == expected_result def test_swh_id_resolve_invalid(api_client): - rev_id_invalid = '96db9023b8_foo_50d6c108e9a3' - swh_id = 'swh:1:rev:%s' % rev_id_invalid - url = reverse('api-1-resolve-swh-pid', url_args={'swh_id': swh_id}) + rev_id_invalid = "96db9023b8_foo_50d6c108e9a3" + swh_id = "swh:1:rev:%s" % rev_id_invalid + url = reverse("api-1-resolve-swh-pid", url_args={"swh_id": swh_id}) resp = api_client.get(url) assert resp.status_code == 400, resp.data -@given(unknown_content(), unknown_directory(), unknown_release(), - unknown_revision(), unknown_snapshot()) -def test_swh_id_resolve_not_found(api_client, unknown_content, - unknown_directory, unknown_release, - unknown_revision, unknown_snapshot): - - for obj_type_short, obj_id in (('cnt', unknown_content['sha1_git']), - ('dir', unknown_directory), - ('rel', unknown_release), - ('rev', unknown_revision), - ('snp', unknown_snapshot)): - - swh_id = 'swh:1:%s:%s' % (obj_type_short, obj_id) - - url = reverse('api-1-resolve-swh-pid', url_args={'swh_id': swh_id}) +@given( + unknown_content(), + unknown_directory(), + unknown_release(), + unknown_revision(), + unknown_snapshot(), +) +def test_swh_id_resolve_not_found( + api_client, + unknown_content, + unknown_directory, + unknown_release, + unknown_revision, + unknown_snapshot, +): + + for obj_type_short, obj_id in ( + ("cnt", unknown_content["sha1_git"]), + ("dir", unknown_directory), + ("rel", unknown_release), + ("rev", unknown_revision), + ("snp", unknown_snapshot), + ): + + swh_id = "swh:1:%s:%s" % (obj_type_short, obj_id) + + url = reverse("api-1-resolve-swh-pid", url_args={"swh_id": swh_id}) resp = api_client.get(url) assert resp.status_code == 404, resp.data def test_swh_origin_id_not_resolvable(api_client): - ori_pid = 'swh:1:ori:8068d0075010b590762c6cb5682ed53cb3c13deb' - url = reverse('api-1-resolve-swh-pid', url_args={'swh_id': ori_pid}) + ori_pid = "swh:1:ori:8068d0075010b590762c6cb5682ed53cb3c13deb" + url = reverse("api-1-resolve-swh-pid", url_args={"swh_id": ori_pid}) resp = api_client.get(url) assert resp.status_code == 400, resp.data @given(content(), directory()) def test_api_known_swhpid_some_present(api_client, content, directory): - content_ = 'swh:1:cnt:%s' % content['sha1_git'] - directory_ = 'swh:1:dir:%s' % directory - unknown_revision_ = 'swh:1:rev:%s' % random_sha1() - unknown_release_ = 'swh:1:rel:%s' % random_sha1() - unknown_snapshot_ = 'swh:1:snp:%s' % random_sha1() - - input_pids = [content_, directory_, unknown_revision_, - unknown_release_, unknown_snapshot_] - - url = reverse('api-1-known') - - resp = api_client.post(url, data=input_pids, format='json', - HTTP_ACCEPT='application/json') + content_ = "swh:1:cnt:%s" % content["sha1_git"] + directory_ = "swh:1:dir:%s" % directory + unknown_revision_ = "swh:1:rev:%s" % random_sha1() + unknown_release_ = "swh:1:rel:%s" % random_sha1() + unknown_snapshot_ = "swh:1:snp:%s" % random_sha1() + + input_pids = [ + content_, + directory_, + unknown_revision_, + unknown_release_, + unknown_snapshot_, + ] + + url = reverse("api-1-known") + + resp = api_client.post( + url, data=input_pids, format="json", HTTP_ACCEPT="application/json" + ) assert resp.status_code == 200, resp.data - assert resp['Content-Type'] == 'application/json' + assert resp["Content-Type"] == "application/json" assert resp.data == { - content_: {'known': True}, - directory_: {'known': True}, - unknown_revision_: {'known': False}, - unknown_release_: {'known': False}, - unknown_snapshot_: {'known': False} - } + content_: {"known": True}, + directory_: {"known": True}, + unknown_revision_: {"known": False}, + unknown_release_: {"known": False}, + unknown_snapshot_: {"known": False}, + } def test_api_known_invalid_swhpid(api_client): - invalid_pid_sha1 = ['swh:1:cnt:8068d0075010b590762c6cb5682ed53cb3c13de;'] - invalid_pid_type = ['swh:1:cnn:8068d0075010b590762c6cb5682ed53cb3c13deb'] + invalid_pid_sha1 = ["swh:1:cnt:8068d0075010b590762c6cb5682ed53cb3c13de;"] + invalid_pid_type = ["swh:1:cnn:8068d0075010b590762c6cb5682ed53cb3c13deb"] - url = reverse('api-1-known') + url = reverse("api-1-known") - resp = api_client.post(url, data=invalid_pid_sha1, format='json', - HTTP_ACCEPT='application/json') + resp = api_client.post( + url, data=invalid_pid_sha1, format="json", HTTP_ACCEPT="application/json" + ) assert resp.status_code == 400, resp.data - resp2 = api_client.post(url, data=invalid_pid_type, format='json', - HTTP_ACCEPT='application/json') + resp2 = api_client.post( + url, data=invalid_pid_type, format="json", HTTP_ACCEPT="application/json" + ) assert resp2.status_code == 400, resp.data def test_api_known_raises_large_payload_error(api_client): - random_pid = 'swh:1:cnt:8068d0075010b590762c6cb5682ed53cb3c13deb' + random_pid = "swh:1:cnt:8068d0075010b590762c6cb5682ed53cb3c13deb" limit = 10000 - err_msg = 'The maximum number of PIDs this endpoint can receive is 1000' + err_msg = "The maximum number of PIDs this endpoint can receive is 1000" pids = [random_pid for i in range(limit)] - url = reverse('api-1-known') - resp = api_client.post(url, data=pids, format='json', - HTTP_ACCEPT='application/json') + url = reverse("api-1-known") + resp = api_client.post( + url, data=pids, format="json", HTTP_ACCEPT="application/json" + ) assert resp.status_code == 413, resp.data - assert resp['Content-Type'] == 'application/json' - assert resp.data == { - 'exception': 'LargePayloadExc', - 'reason': err_msg - } + assert resp["Content-Type"] == "application/json" + assert resp.data == {"exception": "LargePayloadExc", "reason": err_msg} diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py index 5b3e0920..f484d096 100644 --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -1,677 +1,709 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import given import pytest from requests.utils import parse_header_links from swh.model.model import Origin from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api.utils import enrich_origin_visit, enrich_origin from swh.web.common.exc import BadInputExc from swh.web.common.utils import reverse from swh.web.common.origin_visits import get_origin_visits -from swh.web.tests.strategies import ( - origin, new_origin, visit_dates, new_snapshots -) +from swh.web.tests.strategies import origin, new_origin, visit_dates, new_snapshots def _scroll_results(api_client, url): """Iterates through pages of results, and returns them all.""" results = [] while True: rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" results.extend(rv.data) - if 'Link' in rv: - for link in parse_header_links(rv['Link']): - if link['rel'] == 'next': + if "Link" in rv: + for link in parse_header_links(rv["Link"]): + if link["rel"] == "next": # Found link to next page of results - url = link['url'] + url = link["url"] break else: # No link with 'rel=next' break else: # No Link header break return results def test_api_lookup_origin_visits_raise_error(api_client, mocker): - mock_get_origin_visits = mocker.patch( - 'swh.web.api.views.origin.get_origin_visits') - err_msg = 'voluntary error to check the bad request middleware.' + mock_get_origin_visits = mocker.patch("swh.web.api.views.origin.get_origin_visits") + err_msg = "voluntary error to check the bad request middleware." mock_get_origin_visits.side_effect = BadInputExc(err_msg) - url = reverse('api-1-origin-visits', url_args={'origin_url': 'http://foo'}) + url = reverse("api-1-origin-visits", url_args={"origin_url": "http://foo"}) rv = api_client.get(url) assert rv.status_code == 400, rv.data - assert rv['Content-Type'] == 'application/json' - assert rv.data == { - 'exception': 'BadInputExc', - 'reason': err_msg - } + assert rv["Content-Type"] == "application/json" + assert rv.data == {"exception": "BadInputExc", "reason": err_msg} -def test_api_lookup_origin_visits_raise_swh_storage_error_db(api_client, - mocker): - mock_get_origin_visits = mocker.patch( - 'swh.web.api.views.origin.get_origin_visits') - err_msg = 'Storage exploded! Will be back online shortly!' +def test_api_lookup_origin_visits_raise_swh_storage_error_db(api_client, mocker): + mock_get_origin_visits = mocker.patch("swh.web.api.views.origin.get_origin_visits") + err_msg = "Storage exploded! Will be back online shortly!" mock_get_origin_visits.side_effect = StorageDBError(err_msg) - url = reverse('api-1-origin-visits', url_args={'origin_url': 'http://foo'}) + url = reverse("api-1-origin-visits", url_args={"origin_url": "http://foo"}) rv = api_client.get(url) assert rv.status_code == 503, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'StorageDBError', - 'reason': - 'An unexpected error occurred in the backend: %s' % err_msg + "exception": "StorageDBError", + "reason": "An unexpected error occurred in the backend: %s" % err_msg, } -def test_api_lookup_origin_visits_raise_swh_storage_error_api(api_client, - mocker): - mock_get_origin_visits = mocker.patch( - 'swh.web.api.views.origin.get_origin_visits') - err_msg = 'Storage API dropped dead! Will resurrect asap!' +def test_api_lookup_origin_visits_raise_swh_storage_error_api(api_client, mocker): + mock_get_origin_visits = mocker.patch("swh.web.api.views.origin.get_origin_visits") + err_msg = "Storage API dropped dead! Will resurrect asap!" mock_get_origin_visits.side_effect = StorageAPIError(err_msg) - url = reverse( - 'api-1-origin-visits', url_args={'origin_url': 'http://foo'}) + url = reverse("api-1-origin-visits", url_args={"origin_url": "http://foo"}) rv = api_client.get(url) assert rv.status_code == 503, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'StorageAPIError', - 'reason': - 'An unexpected error occurred in the api backend: %s' % err_msg + "exception": "StorageAPIError", + "reason": "An unexpected error occurred in the api backend: %s" % err_msg, } @given(new_origin(), visit_dates(3), new_snapshots(3)) -def test_api_lookup_origin_visits(api_client, archive_data, new_origin, - visit_dates, new_snapshots): +def test_api_lookup_origin_visits( + api_client, archive_data, new_origin, visit_dates, new_snapshots +): archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin.url, visit_date, type='git') + new_origin.url, visit_date, type="git" + ) archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin.url, origin_visit.visit, - status='full', - snapshot=new_snapshots[i].id) + new_origin.url, + origin_visit.visit, + status="full", + snapshot=new_snapshots[i].id, + ) all_visits = list(reversed(get_origin_visits(new_origin.to_dict()))) for last_visit, expected_visits in ( - (None, all_visits[:2]), - (all_visits[1]['visit'], all_visits[2:])): + (None, all_visits[:2]), + (all_visits[1]["visit"], all_visits[2:]), + ): - url = reverse('api-1-origin-visits', - url_args={'origin_url': new_origin.url}, - query_params={'per_page': 2, - 'last_visit': last_visit}) + url = reverse( + "api-1-origin-visits", + url_args={"origin_url": new_origin.url}, + query_params={"per_page": 2, "last_visit": last_visit}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" for i in range(len(expected_visits)): expected_visits[i] = enrich_origin_visit( - expected_visits[i], with_origin_link=False, - with_origin_visit_link=True, request=rv.wsgi_request) + expected_visits[i], + with_origin_link=False, + with_origin_visit_link=True, + request=rv.wsgi_request, + ) assert rv.data == expected_visits @given(new_origin(), visit_dates(3), new_snapshots(3)) -def test_api_lookup_origin_visits_by_id(api_client, archive_data, new_origin, - visit_dates, new_snapshots): +def test_api_lookup_origin_visits_by_id( + api_client, archive_data, new_origin, visit_dates, new_snapshots +): archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin.url, visit_date, type='git') + new_origin.url, visit_date, type="git" + ) archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin.url, origin_visit.visit, - status='full', - snapshot=new_snapshots[i].id) + new_origin.url, + origin_visit.visit, + status="full", + snapshot=new_snapshots[i].id, + ) all_visits = list(reversed(get_origin_visits(new_origin.to_dict()))) for last_visit, expected_visits in ( - (None, all_visits[:2]), - (all_visits[1]['visit'], all_visits[2:4])): + (None, all_visits[:2]), + (all_visits[1]["visit"], all_visits[2:4]), + ): - url = reverse('api-1-origin-visits', - url_args={'origin_url': new_origin.url}, - query_params={'per_page': 2, - 'last_visit': last_visit}) + url = reverse( + "api-1-origin-visits", + url_args={"origin_url": new_origin.url}, + query_params={"per_page": 2, "last_visit": last_visit}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" for i in range(len(expected_visits)): expected_visits[i] = enrich_origin_visit( - expected_visits[i], with_origin_link=False, - with_origin_visit_link=True, request=rv.wsgi_request) + expected_visits[i], + with_origin_link=False, + with_origin_visit_link=True, + request=rv.wsgi_request, + ) assert rv.data == expected_visits @given(new_origin(), visit_dates(3), new_snapshots(3)) -def test_api_lookup_origin_visit(api_client, archive_data, new_origin, - visit_dates, new_snapshots): +def test_api_lookup_origin_visit( + api_client, archive_data, new_origin, visit_dates, new_snapshots +): archive_data.origin_add_one(new_origin) for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin.url, visit_date, type='git') + new_origin.url, visit_date, type="git" + ) visit_id = origin_visit.visit archive_data.snapshot_add([new_snapshots[i]]) archive_data.origin_visit_update( - new_origin.url, visit_id, - status='full', - snapshot=new_snapshots[i].id) - url = reverse('api-1-origin-visit', - url_args={'origin_url': new_origin.url, - 'visit_id': visit_id}) + new_origin.url, visit_id, status="full", snapshot=new_snapshots[i].id + ) + url = reverse( + "api-1-origin-visit", + url_args={"origin_url": new_origin.url, "visit_id": visit_id}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" - expected_visit = archive_data.origin_visit_get_by( - new_origin.url, visit_id) + expected_visit = archive_data.origin_visit_get_by(new_origin.url, visit_id) expected_visit = enrich_origin_visit( - expected_visit, with_origin_link=True, - with_origin_visit_link=False, request=rv.wsgi_request) + expected_visit, + with_origin_link=True, + with_origin_visit_link=False, + request=rv.wsgi_request, + ) assert rv.data == expected_visit @given(new_origin()) -def test_api_lookup_origin_visit_latest_no_visit(api_client, archive_data, - new_origin): +def test_api_lookup_origin_visit_latest_no_visit(api_client, archive_data, new_origin): archive_data.origin_add_one(new_origin) - url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin.url}) + url = reverse("api-1-origin-visit-latest", url_args={"origin_url": new_origin.url}) rv = api_client.get(url) assert rv.status_code == 404, rv.data assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'No visit for origin %s found' % new_origin.url + "exception": "NotFoundExc", + "reason": "No visit for origin %s found" % new_origin.url, } @given(new_origin(), visit_dates(2), new_snapshots(1)) -def test_api_lookup_origin_visit_latest(api_client, archive_data, new_origin, - visit_dates, new_snapshots): +def test_api_lookup_origin_visit_latest( + api_client, archive_data, new_origin, visit_dates, new_snapshots +): archive_data.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin.url, visit_date, type='git') + new_origin.url, visit_date, type="git" + ) visit_ids.append(origin_visit.visit) archive_data.snapshot_add([new_snapshots[0]]) archive_data.origin_visit_update( - new_origin.url, visit_ids[0], - status='full', - snapshot=new_snapshots[0].id) + new_origin.url, visit_ids[0], status="full", snapshot=new_snapshots[0].id + ) - url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin.url}) + url = reverse("api-1-origin-visit-latest", url_args={"origin_url": new_origin.url}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" - expected_visit = archive_data.origin_visit_get_by( - new_origin.url, visit_ids[1]) + expected_visit = archive_data.origin_visit_get_by(new_origin.url, visit_ids[1]) expected_visit = enrich_origin_visit( - expected_visit, with_origin_link=True, - with_origin_visit_link=False, request=rv.wsgi_request) + expected_visit, + with_origin_link=True, + with_origin_visit_link=False, + request=rv.wsgi_request, + ) assert rv.data == expected_visit @given(new_origin(), visit_dates(2), new_snapshots(1)) -def test_api_lookup_origin_visit_latest_with_snapshot(api_client, archive_data, - new_origin, visit_dates, - new_snapshots): +def test_api_lookup_origin_visit_latest_with_snapshot( + api_client, archive_data, new_origin, visit_dates, new_snapshots +): archive_data.origin_add_one(new_origin) visit_dates.sort() visit_ids = [] for i, visit_date in enumerate(visit_dates): origin_visit = archive_data.origin_visit_add( - new_origin.url, visit_date, type='git') + new_origin.url, visit_date, type="git" + ) visit_ids.append(origin_visit.visit) archive_data.snapshot_add([new_snapshots[0]]) archive_data.origin_visit_update( - new_origin.url, visit_ids[0], - status='full', - snapshot=new_snapshots[0].id) + new_origin.url, visit_ids[0], status="full", snapshot=new_snapshots[0].id + ) - url = reverse('api-1-origin-visit-latest', - url_args={'origin_url': new_origin.url}, - query_params={'require_snapshot': True}) + url = reverse( + "api-1-origin-visit-latest", + url_args={"origin_url": new_origin.url}, + query_params={"require_snapshot": True}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" - expected_visit = archive_data.origin_visit_get_by( - new_origin.url, visit_ids[0]) + expected_visit = archive_data.origin_visit_get_by(new_origin.url, visit_ids[0]) expected_visit = enrich_origin_visit( - expected_visit, with_origin_link=True, - with_origin_visit_link=False, request=rv.wsgi_request) + expected_visit, + with_origin_link=True, + with_origin_visit_link=False, + request=rv.wsgi_request, + ) assert rv.data == expected_visit @given(origin()) def test_api_lookup_origin_visit_not_found(api_client, origin): all_visits = list(reversed(get_origin_visits(origin))) - max_visit_id = max([v['visit'] for v in all_visits]) + max_visit_id = max([v["visit"] for v in all_visits]) - url = reverse('api-1-origin-visit', - url_args={'origin_url': origin['url'], - 'visit_id': max_visit_id + 1}) + url = reverse( + "api-1-origin-visit", + url_args={"origin_url": origin["url"], "visit_id": max_visit_id + 1}, + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Origin %s or its visit with id %s not found!' % - (origin['url'], max_visit_id+1) + "exception": "NotFoundExc", + "reason": "Origin %s or its visit with id %s not found!" + % (origin["url"], max_visit_id + 1), } def test_api_origins(api_client, archive_data): origins = list(archive_data.origin_get_range(0, 10000)) - origin_urls = {origin['url'] for origin in origins} + origin_urls = {origin["url"] for origin in origins} # Get only one - url = reverse('api-1-origins', - query_params={'origin_count': 1}) + url = reverse("api-1-origins", query_params={"origin_count": 1}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - assert {origin['url'] for origin in rv.data} <= origin_urls + assert {origin["url"] for origin in rv.data} <= origin_urls # Get all - url = reverse('api-1-origins', - query_params={'origin_count': len(origins)}) + url = reverse("api-1-origins", query_params={"origin_count": len(origins)}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == len(origins) - assert {origin['url'] for origin in rv.data} == origin_urls + assert {origin["url"] for origin in rv.data} == origin_urls # Get "all + 10" - url = reverse('api-1-origins', - query_params={'origin_count': len(origins)+10}) + url = reverse("api-1-origins", query_params={"origin_count": len(origins) + 10}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == len(origins) - assert {origin['url'] for origin in rv.data} == origin_urls + assert {origin["url"] for origin in rv.data} == origin_urls -@pytest.mark.parametrize('origin_count', [1, 2, 10, 100]) +@pytest.mark.parametrize("origin_count", [1, 2, 10, 100]) def test_api_origins_scroll(api_client, archive_data, origin_count): origins = list(archive_data.origin_get_range(0, 10000)) - origin_urls = {origin['url'] for origin in origins} + origin_urls = {origin["url"] for origin in origins} - url = reverse('api-1-origins', - query_params={'origin_count': origin_count}) + url = reverse("api-1-origins", query_params={"origin_count": origin_count}) results = _scroll_results(api_client, url) assert len(results) == len(origins) - assert {origin['url'] for origin in results} == origin_urls + assert {origin["url"] for origin in results} == origin_urls @given(origin()) def test_api_origin_by_url(api_client, archive_data, origin): - url = reverse('api-1-origin', - url_args={'origin_url': origin['url']}) + url = reverse("api-1-origin", url_args={"origin_url": origin["url"]}) rv = api_client.get(url) expected_origin = archive_data.origin_get(origin) expected_origin = enrich_origin(expected_origin, rv.wsgi_request) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_origin @given(new_origin()) def test_api_origin_not_found(api_client, new_origin): - url = reverse('api-1-origin', - url_args={'origin_url': new_origin.url}) + url = reverse("api-1-origin", url_args={"origin_url": new_origin.url}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Origin with url %s not found!' % new_origin.url + "exception": "NotFoundExc", + "reason": "Origin with url %s not found!" % new_origin.url, } -@pytest.mark.parametrize('backend', ['swh-search', 'swh-storage']) +@pytest.mark.parametrize("backend", ["swh-search", "swh-storage"]) def test_api_origin_search(api_client, mocker, backend): - if backend != 'swh-search': + if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch('swh.web.common.service.search', None) + mocker.patch("swh.web.common.service.search", None) expected_origins = { - 'https://github.com/wcoder/highlightjs-line-numbers.js', - 'https://github.com/memononen/libtess2', + "https://github.com/wcoder/highlightjs-line-numbers.js", + "https://github.com/memononen/libtess2", } # Search for 'github.com', get only one - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'github.com'}, - query_params={'limit': 1}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "github.com"}, + query_params={"limit": 1}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - assert {origin['url'] for origin in rv.data} <= expected_origins + assert {origin["url"] for origin in rv.data} <= expected_origins # Search for 'github.com', get all - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'github.com'}, - query_params={'limit': 2}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "github.com"}, + query_params={"limit": 2}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - assert {origin['url'] for origin in rv.data} == expected_origins + assert rv["Content-Type"] == "application/json" + assert {origin["url"] for origin in rv.data} == expected_origins # Search for 'github.com', get more than available - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'github.com'}, - query_params={'limit': 10}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "github.com"}, + query_params={"limit": 10}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - assert {origin['url'] for origin in rv.data} == expected_origins + assert rv["Content-Type"] == "application/json" + assert {origin["url"] for origin in rv.data} == expected_origins -@pytest.mark.parametrize('backend', ['swh-search', 'swh-storage']) +@pytest.mark.parametrize("backend", ["swh-search", "swh-storage"]) def test_api_origin_search_words(api_client, mocker, backend): - if backend != 'swh-search': + if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch('swh.web.common.service.search', None) + mocker.patch("swh.web.common.service.search", None) expected_origins = { - 'https://github.com/wcoder/highlightjs-line-numbers.js', - 'https://github.com/memononen/libtess2', + "https://github.com/wcoder/highlightjs-line-numbers.js", + "https://github.com/memononen/libtess2", } - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'github com'}, - query_params={'limit': 2}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "github com"}, + query_params={"limit": 2}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - assert {origin['url'] for origin in rv.data} == expected_origins + assert rv["Content-Type"] == "application/json" + assert {origin["url"] for origin in rv.data} == expected_origins - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'com github'}, - query_params={'limit': 2}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "com github"}, + query_params={"limit": 2}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - assert {origin['url'] for origin in rv.data} == expected_origins + assert rv["Content-Type"] == "application/json" + assert {origin["url"] for origin in rv.data} == expected_origins - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'memononen libtess2'}, - query_params={'limit': 2}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "memononen libtess2"}, + query_params={"limit": 2}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - assert {origin['url'] for origin in rv.data} \ - == {'https://github.com/memononen/libtess2'} + assert {origin["url"] for origin in rv.data} == { + "https://github.com/memononen/libtess2" + } - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'libtess2 memononen'}, - query_params={'limit': 2}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "libtess2 memononen"}, + query_params={"limit": 2}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - assert {origin['url'] for origin in rv.data} \ - == {'https://github.com/memononen/libtess2'} + assert {origin["url"] for origin in rv.data} == { + "https://github.com/memononen/libtess2" + } -@pytest.mark.parametrize('backend', ['swh-search', 'swh-storage']) -@pytest.mark.parametrize('limit', [1, 2, 3, 10]) -def test_api_origin_search_scroll( - api_client, archive_data, mocker, limit, backend): +@pytest.mark.parametrize("backend", ["swh-search", "swh-storage"]) +@pytest.mark.parametrize("limit", [1, 2, 3, 10]) +def test_api_origin_search_scroll(api_client, archive_data, mocker, limit, backend): - if backend != 'swh-search': + if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch('swh.web.common.service.search', None) + mocker.patch("swh.web.common.service.search", None) expected_origins = { - 'https://github.com/wcoder/highlightjs-line-numbers.js', - 'https://github.com/memononen/libtess2', + "https://github.com/wcoder/highlightjs-line-numbers.js", + "https://github.com/memononen/libtess2", } - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'github.com'}, - query_params={'limit': limit}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "github.com"}, + query_params={"limit": limit}, + ) results = _scroll_results(api_client, url) - assert {origin['url'] for origin in results} == expected_origins + assert {origin["url"] for origin in results} == expected_origins -@pytest.mark.parametrize('backend', ['swh-search', 'swh-storage']) -def test_api_origin_search_limit( - api_client, archive_data, tests_data, mocker, backend): - if backend == 'swh-search': - tests_data['search'].origin_update([ - {'url': 'http://foobar/{}'.format(i)} - for i in range(2000) - ]) +@pytest.mark.parametrize("backend", ["swh-search", "swh-storage"]) +def test_api_origin_search_limit(api_client, archive_data, tests_data, mocker, backend): + if backend == "swh-search": + tests_data["search"].origin_update( + [{"url": "http://foobar/{}".format(i)} for i in range(2000)] + ) else: # equivalent to not configuring search in the config - mocker.patch('swh.web.common.service.search', None) + mocker.patch("swh.web.common.service.search", None) - archive_data.origin_add([ - Origin(url='http://foobar/{}'.format(i)) - for i in range(2000) - ]) + archive_data.origin_add( + [Origin(url="http://foobar/{}".format(i)) for i in range(2000)] + ) - url = reverse('api-1-origin-search', - url_args={'url_pattern': 'foobar'}, - query_params={'limit': 1050}) + url = reverse( + "api-1-origin-search", + url_args={"url_pattern": "foobar"}, + query_params={"limit": 1050}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1000 @given(origin()) def test_api_origin_metadata_search(api_client, mocker, origin): - mock_idx_storage = mocker.patch('swh.web.common.service.idx_storage') + mock_idx_storage = mocker.patch("swh.web.common.service.idx_storage") oimsft = mock_idx_storage.origin_intrinsic_metadata_search_fulltext - oimsft.side_effect = lambda conjunction, limit: [{ - 'from_revision': ( - b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' - b'\xf2U\xfa\x05B8'), - 'metadata': {'author': 'Jane Doe'}, - 'id': origin['url'], - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' + oimsft.side_effect = lambda conjunction, limit: [ + { + "from_revision": ( + b"p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed " b"\xf2U\xfa\x05B8" + ), + "metadata": {"author": "Jane Doe"}, + "id": origin["url"], + "tool": { + "configuration": { + "context": ["NpmMapping", "CodemetaMapping"], + "type": "local", + }, + "id": 3, + "name": "swh-metadata-detector", + "version": "0.0.1", }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1' } - }] + ] - url = reverse('api-1-origin-metadata-search', - query_params={'fulltext': 'Jane Doe'}) + url = reverse("api-1-origin-metadata-search", query_params={"fulltext": "Jane Doe"}) rv = api_client.get(url) assert rv.status_code == 200, rv.content - assert rv['Content-Type'] == 'application/json' - expected_data = [{ - 'url': origin['url'], - 'metadata': { - 'metadata': {'author': 'Jane Doe'}, - 'from_revision': ( - '7026b7c1a2af56521e951c01ed20f255fa054238'), - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' + assert rv["Content-Type"] == "application/json" + expected_data = [ + { + "url": origin["url"], + "metadata": { + "metadata": {"author": "Jane Doe"}, + "from_revision": ("7026b7c1a2af56521e951c01ed20f255fa054238"), + "tool": { + "configuration": { + "context": ["NpmMapping", "CodemetaMapping"], + "type": "local", + }, + "id": 3, + "name": "swh-metadata-detector", + "version": "0.0.1", }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1', - } + }, } - }] + ] assert rv.data == expected_data - oimsft.assert_called_with(conjunction=['Jane Doe'], limit=70) + oimsft.assert_called_with(conjunction=["Jane Doe"], limit=70) @given(origin()) def test_api_origin_metadata_search_limit(api_client, mocker, origin): - mock_idx_storage = mocker.patch('swh.web.common.service.idx_storage') + mock_idx_storage = mocker.patch("swh.web.common.service.idx_storage") oimsft = mock_idx_storage.origin_intrinsic_metadata_search_fulltext - oimsft.side_effect = lambda conjunction, limit: [{ - 'from_revision': ( - b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' - b'\xf2U\xfa\x05B8'), - 'metadata': {'author': 'Jane Doe'}, - 'id': origin['url'], - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' + oimsft.side_effect = lambda conjunction, limit: [ + { + "from_revision": ( + b"p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed " b"\xf2U\xfa\x05B8" + ), + "metadata": {"author": "Jane Doe"}, + "id": origin["url"], + "tool": { + "configuration": { + "context": ["NpmMapping", "CodemetaMapping"], + "type": "local", + }, + "id": 3, + "name": "swh-metadata-detector", + "version": "0.0.1", }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1' } - }] + ] - url = reverse('api-1-origin-metadata-search', - query_params={'fulltext': 'Jane Doe'}) + url = reverse("api-1-origin-metadata-search", query_params={"fulltext": "Jane Doe"}) rv = api_client.get(url) assert rv.status_code == 200, rv.content - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - oimsft.assert_called_with(conjunction=['Jane Doe'], limit=70) + oimsft.assert_called_with(conjunction=["Jane Doe"], limit=70) - url = reverse('api-1-origin-metadata-search', - query_params={'fulltext': 'Jane Doe', - 'limit': 10}) + url = reverse( + "api-1-origin-metadata-search", + query_params={"fulltext": "Jane Doe", "limit": 10}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.content - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - oimsft.assert_called_with(conjunction=['Jane Doe'], limit=10) + oimsft.assert_called_with(conjunction=["Jane Doe"], limit=10) - url = reverse('api-1-origin-metadata-search', - query_params={'fulltext': 'Jane Doe', - 'limit': 987}) + url = reverse( + "api-1-origin-metadata-search", + query_params={"fulltext": "Jane Doe", "limit": 987}, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.content - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert len(rv.data) == 1 - oimsft.assert_called_with(conjunction=['Jane Doe'], limit=100) + oimsft.assert_called_with(conjunction=["Jane Doe"], limit=100) @given(origin()) def test_api_origin_intrinsic_metadata(api_client, mocker, origin): - mock_idx_storage = mocker.patch('swh.web.common.service.idx_storage') + mock_idx_storage = mocker.patch("swh.web.common.service.idx_storage") oimg = mock_idx_storage.origin_intrinsic_metadata_get - oimg.side_effect = lambda origin_urls: [{ - 'from_revision': ( - b'p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed ' - b'\xf2U\xfa\x05B8'), - 'metadata': {'author': 'Jane Doe'}, - 'id': origin['url'], - 'tool': { - 'configuration': { - 'context': ['NpmMapping', 'CodemetaMapping'], - 'type': 'local' + oimg.side_effect = lambda origin_urls: [ + { + "from_revision": ( + b"p&\xb7\xc1\xa2\xafVR\x1e\x95\x1c\x01\xed " b"\xf2U\xfa\x05B8" + ), + "metadata": {"author": "Jane Doe"}, + "id": origin["url"], + "tool": { + "configuration": { + "context": ["NpmMapping", "CodemetaMapping"], + "type": "local", + }, + "id": 3, + "name": "swh-metadata-detector", + "version": "0.0.1", }, - 'id': 3, - 'name': 'swh-metadata-detector', - 'version': '0.0.1' } - }] + ] - url = reverse('api-origin-intrinsic-metadata', - url_args={'origin_url': origin['url']}) + url = reverse( + "api-origin-intrinsic-metadata", url_args={"origin_url": origin["url"]} + ) rv = api_client.get(url) - oimg.assert_called_once_with([origin['url']]) + oimg.assert_called_once_with([origin["url"]]) assert rv.status_code == 200, rv.content - assert rv['Content-Type'] == 'application/json' - expected_data = {'author': 'Jane Doe'} + assert rv["Content-Type"] == "application/json" + expected_data = {"author": "Jane Doe"} assert rv.data == expected_data def test_api_origin_metadata_search_invalid(api_client, mocker): - mock_idx_storage = mocker.patch('swh.web.common.service.idx_storage') - url = reverse('api-1-origin-metadata-search') + mock_idx_storage = mocker.patch("swh.web.common.service.idx_storage") + url = reverse("api-1-origin-metadata-search") rv = api_client.get(url) assert rv.status_code == 400, rv.content mock_idx_storage.assert_not_called() diff --git a/swh/web/tests/api/views/test_origin_save.py b/swh/web/tests/api/views/test_origin_save.py index 7e7727e4..f881f067 100644 --- a/swh/web/tests/api/views/test_origin_save.py +++ b/swh/web/tests/api/views/test_origin_save.py @@ -1,251 +1,328 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from datetime import datetime, timedelta from django.utils import timezone from swh.web.common.utils import reverse from swh.web.common.models import ( - SaveUnauthorizedOrigin, SaveOriginRequest, - SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, - SAVE_REQUEST_PENDING + SaveUnauthorizedOrigin, + SaveOriginRequest, + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING, ) from swh.web.common.models import ( - SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, - SAVE_TASK_SCHEDULED, SAVE_TASK_FAILED, SAVE_TASK_SUCCEED + SAVE_TASK_NOT_CREATED, + SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, + SAVE_TASK_FAILED, + SAVE_TASK_SUCCEED, ) pytestmark = pytest.mark.django_db @pytest.fixture(autouse=True) def populated_db(): - SaveUnauthorizedOrigin.objects.create( - url='https://github.com/user/illegal_repo') - SaveUnauthorizedOrigin.objects.create( - url='https://gitlab.com/user_to_exclude') + SaveUnauthorizedOrigin.objects.create(url="https://github.com/user/illegal_repo") + SaveUnauthorizedOrigin.objects.create(url="https://gitlab.com/user_to_exclude") def test_invalid_visit_type(api_client): - url = reverse('api-1-save-origin', - url_args={'visit_type': 'foo', - 'origin_url': 'https://github.com/torvalds/linux'}) + url = reverse( + "api-1-save-origin", + url_args={ + "visit_type": "foo", + "origin_url": "https://github.com/torvalds/linux", + }, + ) response = api_client.post(url) assert response.status_code == 400 def test_invalid_origin_url(api_client): - url = reverse('api-1-save-origin', - url_args={'visit_type': 'git', - 'origin_url': 'bar'}) + url = reverse( + "api-1-save-origin", url_args={"visit_type": "git", "origin_url": "bar"} + ) response = api_client.post(url) assert response.status_code == 400 -def check_created_save_request_status(api_client, mocker, origin_url, - scheduler_task_status, - expected_request_status, - expected_task_status=None, - visit_date=None): +def check_created_save_request_status( + api_client, + mocker, + origin_url, + scheduler_task_status, + expected_request_status, + expected_task_status=None, + visit_date=None, +): - mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') + mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") if not scheduler_task_status: mock_scheduler.get_tasks.return_value = [] else: - mock_scheduler.get_tasks.return_value = [{ - 'priority': 'high', - 'policy': 'oneshot', - 'type': 'load-git', - 'arguments': { - 'kwargs': { - 'repo_url': origin_url - }, - 'args': [] - }, - 'status': scheduler_task_status, - 'id': 1, - }] - - mock_scheduler.create_tasks.return_value = [{ - 'priority': 'high', - 'policy': 'oneshot', - 'type': 'load-git', - 'arguments': { - 'kwargs': { - 'repo_url': origin_url - }, - 'args': [] - }, - 'status': 'next_run_not_scheduled', - 'id': 1, - }] - - url = reverse('api-1-save-origin', - url_args={'visit_type': 'git', - 'origin_url': origin_url}) - - mock_visit_date = mocker.patch(('swh.web.common.origin_save.' - '_get_visit_info_for_save_request')) + mock_scheduler.get_tasks.return_value = [ + { + "priority": "high", + "policy": "oneshot", + "type": "load-git", + "arguments": {"kwargs": {"repo_url": origin_url}, "args": []}, + "status": scheduler_task_status, + "id": 1, + } + ] + + mock_scheduler.create_tasks.return_value = [ + { + "priority": "high", + "policy": "oneshot", + "type": "load-git", + "arguments": {"kwargs": {"repo_url": origin_url}, "args": []}, + "status": "next_run_not_scheduled", + "id": 1, + } + ] + + url = reverse( + "api-1-save-origin", url_args={"visit_type": "git", "origin_url": origin_url} + ) + + mock_visit_date = mocker.patch( + ("swh.web.common.origin_save." "_get_visit_info_for_save_request") + ) mock_visit_date.return_value = (visit_date, None) response = api_client.post(url) if expected_request_status != SAVE_REQUEST_REJECTED: assert response.status_code == 200, response.data - assert (response.data['save_request_status'] == - expected_request_status) - assert response.data['save_task_status'] == expected_task_status + assert response.data["save_request_status"] == expected_request_status + assert response.data["save_task_status"] == expected_task_status else: assert response.status_code == 403, response.data -def check_save_request_status(api_client, mocker, origin_url, - expected_request_status, - expected_task_status, - scheduler_task_status='next_run_not_scheduled', - visit_date=None): - mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') - mock_scheduler.get_tasks.return_value = [{ - 'priority': 'high', - 'policy': 'oneshot', - 'type': 'load-git', - 'arguments': { - 'kwargs': { - 'repo_url': origin_url - }, - 'args': [] - }, - 'status': scheduler_task_status, - 'id': 1, - }] - - url = reverse('api-1-save-origin', - url_args={'visit_type': 'git', - 'origin_url': origin_url}) - - mock_visit_date = mocker.patch(('swh.web.common.origin_save.' - '_get_visit_info_for_save_request')) +def check_save_request_status( + api_client, + mocker, + origin_url, + expected_request_status, + expected_task_status, + scheduler_task_status="next_run_not_scheduled", + visit_date=None, +): + mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") + mock_scheduler.get_tasks.return_value = [ + { + "priority": "high", + "policy": "oneshot", + "type": "load-git", + "arguments": {"kwargs": {"repo_url": origin_url}, "args": []}, + "status": scheduler_task_status, + "id": 1, + } + ] + + url = reverse( + "api-1-save-origin", url_args={"visit_type": "git", "origin_url": origin_url} + ) + + mock_visit_date = mocker.patch( + ("swh.web.common.origin_save." "_get_visit_info_for_save_request") + ) mock_visit_date.return_value = (visit_date, None) response = api_client.get(url) assert response.status_code == 200, response.data save_request_data = response.data[0] - assert (save_request_data['save_request_status'] == - expected_request_status) - assert save_request_data['save_task_status'] == expected_task_status + assert save_request_data["save_request_status"] == expected_request_status + assert save_request_data["save_task_status"] == expected_task_status # Check that save task status is still available when # the scheduler task has been archived mock_scheduler.get_tasks.return_value = [] response = api_client.get(url) assert response.status_code == 200 save_request_data = response.data[0] - assert save_request_data['save_task_status'] == expected_task_status + assert save_request_data["save_task_status"] == expected_task_status def test_save_request_rejected(api_client, mocker): - origin_url = 'https://github.com/user/illegal_repo' - check_created_save_request_status(api_client, mocker, origin_url, - None, SAVE_REQUEST_REJECTED) - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_CREATED) + origin_url = "https://github.com/user/illegal_repo" + check_created_save_request_status( + api_client, mocker, origin_url, None, SAVE_REQUEST_REJECTED + ) + check_save_request_status( + api_client, mocker, origin_url, SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_CREATED + ) def test_save_request_pending(api_client, mocker): - origin_url = 'https://unkwownforge.com/user/repo' - check_created_save_request_status(api_client, mocker, - origin_url, None, SAVE_REQUEST_PENDING, - SAVE_TASK_NOT_CREATED) - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED) + origin_url = "https://unkwownforge.com/user/repo" + check_created_save_request_status( + api_client, + mocker, + origin_url, + None, + SAVE_REQUEST_PENDING, + SAVE_TASK_NOT_CREATED, + ) + check_save_request_status( + api_client, mocker, origin_url, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED + ) def test_save_request_succeed(api_client, mocker): - origin_url = 'https://github.com/Kitware/CMake' - check_created_save_request_status(api_client, mocker, origin_url, - None, SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED) - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, - scheduler_task_status='next_run_scheduled') - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, - scheduler_task_status='completed', - visit_date=None) + origin_url = "https://github.com/Kitware/CMake" + check_created_save_request_status( + api_client, + mocker, + origin_url, + None, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_NOT_YET_SCHEDULED, + ) + check_save_request_status( + api_client, + mocker, + origin_url, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_SCHEDULED, + scheduler_task_status="next_run_scheduled", + ) + check_save_request_status( + api_client, + mocker, + origin_url, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_SUCCEED, + scheduler_task_status="completed", + visit_date=None, + ) visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEED, - scheduler_task_status='completed', - visit_date=visit_date) + check_save_request_status( + api_client, + mocker, + origin_url, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_SUCCEED, + scheduler_task_status="completed", + visit_date=visit_date, + ) def test_save_request_failed(api_client, mocker): - origin_url = 'https://gitlab.com/inkscape/inkscape' - check_created_save_request_status(api_client, mocker, origin_url, - None, SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED) - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, SAVE_TASK_SCHEDULED, - scheduler_task_status='next_run_scheduled') - check_save_request_status(api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, SAVE_TASK_FAILED, - scheduler_task_status='disabled') + origin_url = "https://gitlab.com/inkscape/inkscape" + check_created_save_request_status( + api_client, + mocker, + origin_url, + None, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_NOT_YET_SCHEDULED, + ) + check_save_request_status( + api_client, + mocker, + origin_url, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_SCHEDULED, + scheduler_task_status="next_run_scheduled", + ) + check_save_request_status( + api_client, + mocker, + origin_url, + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_FAILED, + scheduler_task_status="disabled", + ) def test_create_save_request_only_when_needed(api_client, mocker): - origin_url = 'https://github.com/webpack/webpack' - SaveOriginRequest.objects.create(visit_type='git', origin_url=origin_url, - status=SAVE_REQUEST_ACCEPTED, - loading_task_id=56) - - check_created_save_request_status(api_client, mocker, origin_url, - 'next_run_not_scheduled', - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED) - - sors = list(SaveOriginRequest.objects.filter(visit_type='git', - origin_url=origin_url)) + origin_url = "https://github.com/webpack/webpack" + SaveOriginRequest.objects.create( + visit_type="git", + origin_url=origin_url, + status=SAVE_REQUEST_ACCEPTED, + loading_task_id=56, + ) + + check_created_save_request_status( + api_client, + mocker, + origin_url, + "next_run_not_scheduled", + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_NOT_YET_SCHEDULED, + ) + + sors = list( + SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) + ) assert len(sors) == 1 - check_created_save_request_status(api_client, mocker, origin_url, - 'next_run_scheduled', - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SCHEDULED) - sors = list(SaveOriginRequest.objects.filter(visit_type='git', - origin_url=origin_url)) + check_created_save_request_status( + api_client, + mocker, + origin_url, + "next_run_scheduled", + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_SCHEDULED, + ) + sors = list( + SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) + ) assert len(sors) == 1 visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) - check_created_save_request_status(api_client, mocker, origin_url, - 'completed', SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, - visit_date=visit_date) - sors = list(SaveOriginRequest.objects.filter(visit_type='git', - origin_url=origin_url)) + check_created_save_request_status( + api_client, + mocker, + origin_url, + "completed", + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_NOT_YET_SCHEDULED, + visit_date=visit_date, + ) + sors = list( + SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) + ) assert len(sors) == 2 - check_created_save_request_status(api_client, mocker, origin_url, - 'disabled', SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED) - sors = list(SaveOriginRequest.objects.filter(visit_type='git', - origin_url=origin_url)) + check_created_save_request_status( + api_client, + mocker, + origin_url, + "disabled", + SAVE_REQUEST_ACCEPTED, + SAVE_TASK_NOT_YET_SCHEDULED, + ) + sors = list( + SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) + ) assert len(sors) == 3 def test_get_save_requests_unknown_origin(api_client): - unknown_origin_url = 'https://gitlab.com/foo/bar' - url = reverse('api-1-save-origin', - url_args={'visit_type': 'git', - 'origin_url': unknown_origin_url}) + unknown_origin_url = "https://gitlab.com/foo/bar" + url = reverse( + "api-1-save-origin", + url_args={"visit_type": "git", "origin_url": unknown_origin_url}, + ) response = api_client.get(url) assert response.status_code == 404 assert response.data == { - 'exception': 'NotFoundExc', - 'reason': ('No save requests found for visit of type ' - 'git on origin with url %s.') % unknown_origin_url + "exception": "NotFoundExc", + "reason": ( + "No save requests found for visit of type " "git on origin with url %s." + ) + % unknown_origin_url, } diff --git a/swh/web/tests/api/views/test_ping.py b/swh/web/tests/api/views/test_ping.py index 80659db4..36408bef 100644 --- a/swh/web/tests/api/views/test_ping.py +++ b/swh/web/tests/api/views/test_ping.py @@ -1,16 +1,16 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.utils import reverse def test_api_1_ping(api_client): - url = reverse('api-1-ping') + url = reverse("api-1-ping") rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' - assert rv.data == 'pong' + assert rv["Content-Type"] == "application/json" + assert rv.data == "pong" diff --git a/swh/web/tests/api/views/test_release.py b/swh/web/tests/api/views/test_release.py index de4551bb..3c7b259b 100644 --- a/swh/web/tests/api/views/test_release.py +++ b/swh/web/tests/api/views/test_release.py @@ -1,120 +1,128 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from hypothesis import given from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.model import ( - ObjectType, Person, Release, Timestamp, TimestampWithTimezone + ObjectType, + Person, + Release, + Timestamp, + TimestampWithTimezone, ) from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.strategies import ( - release, content, directory -) +from swh.web.tests.strategies import release, content, directory @given(release()) def test_api_release(api_client, archive_data, release): - url = reverse('api-1-release', url_args={'sha1_git': release}) + url = reverse("api-1-release", url_args={"sha1_git": release}) rv = api_client.get(url) expected_release = archive_data.release_get(release) - target_revision = expected_release['target'] - target_url = reverse('api-1-revision', - url_args={'sha1_git': target_revision}, - request=rv.wsgi_request) - expected_release['target_url'] = target_url + target_revision = expected_release["target"] + target_url = reverse( + "api-1-revision", + url_args={"sha1_git": target_revision}, + request=rv.wsgi_request, + ) + expected_release["target_url"] = target_url assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_release @given(content(), directory(), release()) -def test_api_release_target_type_not_a_revision(api_client, archive_data, - content, directory, release): - for target_type, target in ((ObjectType.CONTENT, content), - (ObjectType.DIRECTORY, directory), - (ObjectType.RELEASE, release)): +def test_api_release_target_type_not_a_revision( + api_client, archive_data, content, directory, release +): + for target_type, target in ( + (ObjectType.CONTENT, content), + (ObjectType.DIRECTORY, directory), + (ObjectType.RELEASE, release), + ): if target_type == ObjectType.CONTENT: - target = target['sha1_git'] + target = target["sha1_git"] sample_release = Release( author=Person( - email=b'author@company.org', - fullname=b'author ', - name=b'author' + email=b"author@company.org", + fullname=b"author ", + name=b"author", ), date=TimestampWithTimezone( timestamp=Timestamp( - seconds=int(datetime.now().timestamp()), - microseconds=0), + seconds=int(datetime.now().timestamp()), microseconds=0 + ), offset=0, negative_utc=False, ), - message=b'sample release message', - name=b'sample release', + message=b"sample release message", + name=b"sample release", synthetic=False, target=hash_to_bytes(target), - target_type=target_type + target_type=target_type, ) archive_data.release_add([sample_release]) new_release_id = hash_to_hex(sample_release.id) - url = reverse('api-1-release', - url_args={'sha1_git': new_release_id}) + url = reverse("api-1-release", url_args={"sha1_git": new_release_id}) rv = api_client.get(url) expected_release = archive_data.release_get(new_release_id) if target_type == ObjectType.CONTENT: - url_args = {'q': 'sha1_git:%s' % target} + url_args = {"q": "sha1_git:%s" % target} else: - url_args = {'sha1_git': target} + url_args = {"sha1_git": target} - target_url = reverse('api-1-%s' % target_type.value, - url_args=url_args, - request=rv.wsgi_request) - expected_release['target_url'] = target_url + target_url = reverse( + "api-1-%s" % target_type.value, url_args=url_args, request=rv.wsgi_request + ) + expected_release["target_url"] = target_url assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_release def test_api_release_not_found(api_client): unknown_release_ = random_sha1() - url = reverse('api-1-release', url_args={'sha1_git': unknown_release_}) + url = reverse("api-1-release", url_args={"sha1_git": unknown_release_}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Release with sha1_git %s not found.' % unknown_release_ + "exception": "NotFoundExc", + "reason": "Release with sha1_git %s not found." % unknown_release_, } @given(release()) def test_api_release_uppercase(api_client, release): - url = reverse('api-1-release-uppercase-checksum', - url_args={'sha1_git': release.upper()}) + url = reverse( + "api-1-release-uppercase-checksum", url_args={"sha1_git": release.upper()} + ) resp = api_client.get(url) assert resp.status_code == 302 - redirect_url = reverse('api-1-release-uppercase-checksum', - url_args={'sha1_git': release}) + redirect_url = reverse( + "api-1-release-uppercase-checksum", url_args={"sha1_git": release} + ) - assert resp['location'] == redirect_url + assert resp["location"] == redirect_url diff --git a/swh/web/tests/api/views/test_revision.py b/swh/web/tests/api/views/test_revision.py index ab590333..7ee31f2f 100644 --- a/swh/web/tests/api/views/test_revision.py +++ b/swh/web/tests/api/views/test_revision.py @@ -1,259 +1,264 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import given from swh.web.api.utils import enrich_revision from swh.web.common.exc import NotFoundExc from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 from swh.web.tests.strategies import revision @given(revision()) def test_api_revision(api_client, archive_data, revision): - url = reverse('api-1-revision', url_args={'sha1_git': revision}) + url = reverse("api-1-revision", url_args={"sha1_git": revision}) rv = api_client.get(url) expected_revision = archive_data.revision_get(revision) enrich_revision(expected_revision, rv.wsgi_request) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_revision def test_api_revision_not_found(api_client): unknown_revision_ = random_sha1() - url = reverse('api-1-revision', - url_args={'sha1_git': unknown_revision_}) + url = reverse("api-1-revision", url_args={"sha1_git": unknown_revision_}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Revision with sha1_git %s not found.' % unknown_revision_ + "exception": "NotFoundExc", + "reason": "Revision with sha1_git %s not found." % unknown_revision_, } @given(revision()) def test_api_revision_raw_ok(api_client, archive_data, revision): - url = reverse('api-1-revision-raw-message', - url_args={'sha1_git': revision}) + url = reverse("api-1-revision-raw-message", url_args={"sha1_git": revision}) rv = api_client.get(url) - expected_message = archive_data.revision_get(revision)['message'] + expected_message = archive_data.revision_get(revision)["message"] assert rv.status_code == 200 - assert rv['Content-Type'] == 'application/octet-stream' + assert rv["Content-Type"] == "application/octet-stream" assert rv.content == expected_message.encode() def test_api_revision_raw_ko_no_rev(api_client): unknown_revision_ = random_sha1() - url = reverse('api-1-revision-raw-message', - url_args={'sha1_git': unknown_revision_}) + url = reverse( + "api-1-revision-raw-message", url_args={"sha1_git": unknown_revision_} + ) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Revision with sha1_git %s not found.' % unknown_revision_ + "exception": "NotFoundExc", + "reason": "Revision with sha1_git %s not found." % unknown_revision_, } @given(revision()) def test_api_revision_log(api_client, archive_data, revision): per_page = 10 - url = reverse('api-1-revision-log', url_args={'sha1_git': revision}, - query_params={'per_page': per_page}) + url = reverse( + "api-1-revision-log", + url_args={"sha1_git": revision}, + query_params={"per_page": per_page}, + ) rv = api_client.get(url) - expected_log = archive_data.revision_log(revision, limit=per_page+1) - expected_log = list(map(enrich_revision, expected_log, - [rv.wsgi_request] * len(expected_log))) + expected_log = archive_data.revision_log(revision, limit=per_page + 1) + expected_log = list( + map(enrich_revision, expected_log, [rv.wsgi_request] * len(expected_log)) + ) has_next = len(expected_log) > per_page assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == (expected_log[:-1] if has_next else expected_log) if has_next: - assert 'Link' in rv + assert "Link" in rv next_log_url = rv.wsgi_request.build_absolute_uri( - reverse('api-1-revision-log', - url_args={'sha1_git': expected_log[-1]['id']}, - query_params={'per_page': per_page})) - assert next_log_url in rv['Link'] + reverse( + "api-1-revision-log", + url_args={"sha1_git": expected_log[-1]["id"]}, + query_params={"per_page": per_page}, + ) + ) + assert next_log_url in rv["Link"] def test_api_revision_log_not_found(api_client): unknown_revision_ = random_sha1() - url = reverse('api-1-revision-log', - url_args={'sha1_git': unknown_revision_}) + url = reverse("api-1-revision-log", url_args={"sha1_git": unknown_revision_}) rv = api_client.get(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Revision with sha1_git %s not found.' % unknown_revision_ + "exception": "NotFoundExc", + "reason": "Revision with sha1_git %s not found." % unknown_revision_, } - assert not rv.has_header('Link') + assert not rv.has_header("Link") @given(revision()) def test_api_revision_log_context(api_client, archive_data, revision): revisions = archive_data.revision_log(revision, limit=4) - prev_rev = revisions[0]['id'] - rev = revisions[-1]['id'] + prev_rev = revisions[0]["id"] + rev = revisions[-1]["id"] per_page = 10 - url = reverse('api-1-revision-log', - url_args={'sha1_git': rev, - 'prev_sha1s': prev_rev}, - query_params={'per_page': per_page}) + url = reverse( + "api-1-revision-log", + url_args={"sha1_git": rev, "prev_sha1s": prev_rev}, + query_params={"per_page": per_page}, + ) rv = api_client.get(url) expected_log = archive_data.revision_log(rev, limit=per_page) prev_revision = archive_data.revision_get(prev_rev) expected_log.insert(0, prev_revision) - expected_log = list(map(enrich_revision, expected_log, - [rv.wsgi_request] * len(expected_log))) + expected_log = list( + map(enrich_revision, expected_log, [rv.wsgi_request] * len(expected_log)) + ) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_log def test_api_revision_directory_ko_not_found(api_client, mocker): - mock_rev_dir = mocker.patch( - 'swh.web.api.views.revision._revision_directory_by') - mock_rev_dir.side_effect = NotFoundExc('Not found') + mock_rev_dir = mocker.patch("swh.web.api.views.revision._revision_directory_by") + mock_rev_dir.side_effect = NotFoundExc("Not found") - rv = api_client.get('/api/1/revision/999/directory/some/path/to/dir/') + rv = api_client.get("/api/1/revision/999/directory/some/path/to/dir/") assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' - assert rv.data == { - 'exception': 'NotFoundExc', - 'reason': 'Not found' - } + assert rv["Content-Type"] == "application/json" + assert rv.data == {"exception": "NotFoundExc", "reason": "Not found"} mock_rev_dir.assert_called_once_with( - {'sha1_git': '999'}, - 'some/path/to/dir', - '/api/1/revision/999/directory/some/path/to/dir/', - with_data=False + {"sha1_git": "999"}, + "some/path/to/dir", + "/api/1/revision/999/directory/some/path/to/dir/", + with_data=False, ) def test_api_revision_directory_ok_returns_dir_entries(api_client, mocker): - mock_rev_dir = mocker.patch( - 'swh.web.api.views.revision._revision_directory_by') + mock_rev_dir = mocker.patch("swh.web.api.views.revision._revision_directory_by") stub_dir = { - 'type': 'dir', - 'revision': '999', - 'content': [ + "type": "dir", + "revision": "999", + "content": [ { - 'sha1_git': '789', - 'type': 'file', - 'target': '101', - 'target_url': '/api/1/content/sha1_git:101/', - 'name': 'somefile', - 'file_url': '/api/1/revision/999/directory/some/path/' - 'somefile/' + "sha1_git": "789", + "type": "file", + "target": "101", + "target_url": "/api/1/content/sha1_git:101/", + "name": "somefile", + "file_url": "/api/1/revision/999/directory/some/path/" "somefile/", }, { - 'sha1_git': '123', - 'type': 'dir', - 'target': '456', - 'target_url': '/api/1/directory/456/', - 'name': 'to-subdir', - 'dir_url': '/api/1/revision/999/directory/some/path/' - 'to-subdir/', - } - ] + "sha1_git": "123", + "type": "dir", + "target": "456", + "target_url": "/api/1/directory/456/", + "name": "to-subdir", + "dir_url": "/api/1/revision/999/directory/some/path/" "to-subdir/", + }, + ], } mock_rev_dir.return_value = stub_dir - rv = api_client.get('/api/1/revision/999/directory/some/path/') + rv = api_client.get("/api/1/revision/999/directory/some/path/") - stub_dir['content'][0]['target_url'] = rv.wsgi_request.build_absolute_uri( - stub_dir['content'][0]['target_url']) - stub_dir['content'][0]['file_url'] = rv.wsgi_request.build_absolute_uri( - stub_dir['content'][0]['file_url']) - stub_dir['content'][1]['target_url'] = rv.wsgi_request.build_absolute_uri( - stub_dir['content'][1]['target_url']) - stub_dir['content'][1]['dir_url'] = rv.wsgi_request.build_absolute_uri( - stub_dir['content'][1]['dir_url']) + stub_dir["content"][0]["target_url"] = rv.wsgi_request.build_absolute_uri( + stub_dir["content"][0]["target_url"] + ) + stub_dir["content"][0]["file_url"] = rv.wsgi_request.build_absolute_uri( + stub_dir["content"][0]["file_url"] + ) + stub_dir["content"][1]["target_url"] = rv.wsgi_request.build_absolute_uri( + stub_dir["content"][1]["target_url"] + ) + stub_dir["content"][1]["dir_url"] = rv.wsgi_request.build_absolute_uri( + stub_dir["content"][1]["dir_url"] + ) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == stub_dir mock_rev_dir.assert_called_once_with( - {'sha1_git': '999'}, - 'some/path', - '/api/1/revision/999/directory/some/path/', - with_data=False + {"sha1_git": "999"}, + "some/path", + "/api/1/revision/999/directory/some/path/", + with_data=False, ) def test_api_revision_directory_ok_returns_content(api_client, mocker): - mock_rev_dir = mocker.patch( - 'swh.web.api.views.revision._revision_directory_by') + mock_rev_dir = mocker.patch("swh.web.api.views.revision._revision_directory_by") stub_content = { - 'type': 'file', - 'revision': '999', - 'content': { - 'sha1_git': '789', - 'sha1': '101', - 'data_url': '/api/1/content/101/raw/', - } + "type": "file", + "revision": "999", + "content": { + "sha1_git": "789", + "sha1": "101", + "data_url": "/api/1/content/101/raw/", + }, } mock_rev_dir.return_value = stub_content - url = '/api/1/revision/666/directory/some/other/path/' + url = "/api/1/revision/666/directory/some/other/path/" rv = api_client.get(url) - stub_content['content']['data_url'] = rv.wsgi_request.build_absolute_uri( - stub_content['content']['data_url']) + stub_content["content"]["data_url"] = rv.wsgi_request.build_absolute_uri( + stub_content["content"]["data_url"] + ) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == stub_content mock_rev_dir.assert_called_once_with( - {'sha1_git': '666'}, 'some/other/path', url, with_data=False) + {"sha1_git": "666"}, "some/other/path", url, with_data=False + ) @given(revision()) def test_api_revision_uppercase(api_client, revision): - url = reverse('api-1-revision-uppercase-checksum', - url_args={'sha1_git': revision.upper()}) + url = reverse( + "api-1-revision-uppercase-checksum", url_args={"sha1_git": revision.upper()} + ) resp = api_client.get(url) assert resp.status_code == 302 - redirect_url = reverse('api-1-revision', - url_args={'sha1_git': revision}) + redirect_url = reverse("api-1-revision", url_args={"sha1_git": revision}) - assert resp['location'] == redirect_url + assert resp["location"] == redirect_url diff --git a/swh/web/tests/api/views/test_snapshot.py b/swh/web/tests/api/views/test_snapshot.py index fabca5ee..25f6e035 100644 --- a/swh/web/tests/api/views/test_snapshot.py +++ b/swh/web/tests/api/views/test_snapshot.py @@ -1,163 +1,165 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import random from hypothesis import given from swh.model.hashutil import hash_to_hex from swh.model.model import Snapshot from swh.web.api.utils import enrich_snapshot from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.strategies import ( - snapshot, new_snapshot -) +from swh.web.tests.strategies import snapshot, new_snapshot @given(snapshot()) def test_api_snapshot(api_client, archive_data, snapshot): - url = reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": snapshot}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" expected_data = archive_data.snapshot_get(snapshot) expected_data = enrich_snapshot(expected_data, rv.wsgi_request) assert rv.data == expected_data @given(snapshot()) def test_api_snapshot_paginated(api_client, archive_data, snapshot): branches_offset = 0 branches_count = 2 snapshot_branches = [] - for k, v in sorted( - archive_data.snapshot_get(snapshot)['branches'].items()): - snapshot_branches.append({ - 'name': k, - 'target_type': v['target_type'], - 'target': v['target'] - }) + for k, v in sorted(archive_data.snapshot_get(snapshot)["branches"].items()): + snapshot_branches.append( + {"name": k, "target_type": v["target_type"], "target": v["target"]} + ) - whole_snapshot = {'id': snapshot, 'branches': {}, 'next_branch': None} + whole_snapshot = {"id": snapshot, "branches": {}, "next_branch": None} while branches_offset < len(snapshot_branches): - branches_from = snapshot_branches[branches_offset]['name'] - url = reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}, - query_params={'branches_from': branches_from, - 'branches_count': branches_count}) + branches_from = snapshot_branches[branches_offset]["name"] + url = reverse( + "api-1-snapshot", + url_args={"snapshot_id": snapshot}, + query_params={ + "branches_from": branches_from, + "branches_count": branches_count, + }, + ) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" expected_data = archive_data.snapshot_get_branches( - snapshot, branches_from, branches_count) + snapshot, branches_from, branches_count + ) expected_data = enrich_snapshot(expected_data, rv.wsgi_request) branches_offset += branches_count if branches_offset < len(snapshot_branches): - next_branch = snapshot_branches[branches_offset]['name'] - expected_data['next_branch'] = next_branch + next_branch = snapshot_branches[branches_offset]["name"] + expected_data["next_branch"] = next_branch else: - expected_data['next_branch'] = None + expected_data["next_branch"] = None assert rv.data == expected_data - whole_snapshot['branches'].update(expected_data['branches']) + whole_snapshot["branches"].update(expected_data["branches"]) if branches_offset < len(snapshot_branches): next_url = rv.wsgi_request.build_absolute_uri( - reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}, - query_params={'branches_from': next_branch, - 'branches_count': branches_count})) - assert rv['Link'] == '<%s>; rel="next"' % next_url + reverse( + "api-1-snapshot", + url_args={"snapshot_id": snapshot}, + query_params={ + "branches_from": next_branch, + "branches_count": branches_count, + }, + ) + ) + assert rv["Link"] == '<%s>; rel="next"' % next_url else: - assert not rv.has_header('Link') + assert not rv.has_header("Link") - url = reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": snapshot}) rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == whole_snapshot @given(snapshot()) def test_api_snapshot_filtered(api_client, archive_data, snapshot): snapshot_branches = [] - for k, v in sorted( - archive_data.snapshot_get(snapshot)['branches'].items()): - snapshot_branches.append({ - 'name': k, - 'target_type': v['target_type'], - 'target': v['target'] - }) + for k, v in sorted(archive_data.snapshot_get(snapshot)["branches"].items()): + snapshot_branches.append( + {"name": k, "target_type": v["target_type"], "target": v["target"]} + ) - target_type = random.choice(snapshot_branches)['target_type'] + target_type = random.choice(snapshot_branches)["target_type"] - url = reverse('api-1-snapshot', - url_args={'snapshot_id': snapshot}, - query_params={'target_types': target_type}) + url = reverse( + "api-1-snapshot", + url_args={"snapshot_id": snapshot}, + query_params={"target_types": target_type}, + ) rv = api_client.get(url) expected_data = archive_data.snapshot_get_branches( - snapshot, target_types=target_type) + snapshot, target_types=target_type + ) expected_data = enrich_snapshot(expected_data, rv.wsgi_request) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == expected_data def test_api_snapshot_errors(api_client): unknown_snapshot_ = random_sha1() - url = reverse('api-1-snapshot', - url_args={'snapshot_id': '63ce369'}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": "63ce369"}) rv = api_client.get(url) assert rv.status_code == 400, rv.data - url = reverse('api-1-snapshot', - url_args={'snapshot_id': unknown_snapshot_}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": unknown_snapshot_}) rv = api_client.get(url) assert rv.status_code == 404, rv.data @given(snapshot()) def test_api_snapshot_uppercase(api_client, snapshot): - url = reverse('api-1-snapshot-uppercase-checksum', - url_args={'snapshot_id': snapshot.upper()}) + url = reverse( + "api-1-snapshot-uppercase-checksum", url_args={"snapshot_id": snapshot.upper()} + ) resp = api_client.get(url) assert resp.status_code == 302 - redirect_url = reverse('api-1-snapshot-uppercase-checksum', - url_args={'snapshot_id': snapshot}) + redirect_url = reverse( + "api-1-snapshot-uppercase-checksum", url_args={"snapshot_id": snapshot} + ) - assert resp['location'] == redirect_url + assert resp["location"] == redirect_url @given(new_snapshot(min_size=4)) def test_api_snapshot_null_branch(api_client, archive_data, new_snapshot): snp_dict = new_snapshot.to_dict() - snp_id = hash_to_hex(snp_dict['id']) - for branch in snp_dict['branches'].keys(): - snp_dict['branches'][branch] = None + snp_id = hash_to_hex(snp_dict["id"]) + for branch in snp_dict["branches"].keys(): + snp_dict["branches"][branch] = None break archive_data.snapshot_add([Snapshot.from_dict(snp_dict)]) - url = reverse('api-1-snapshot', - url_args={'snapshot_id': snp_id}) + url = reverse("api-1-snapshot", url_args={"snapshot_id": snp_id}) rv = api_client.get(url) assert rv.status_code == 200, rv.data diff --git a/swh/web/tests/api/views/test_stat.py b/swh/web/tests/api/views/test_stat.py index 1a300018..c6541a96 100644 --- a/swh/web/tests/api/views/test_stat.py +++ b/swh/web/tests/api/views/test_stat.py @@ -1,70 +1,72 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.common.exc import BadInputExc from swh.web.common.utils import reverse def test_api_1_stat_counters_raise_error(api_client, mocker): - mock_service = mocker.patch('swh.web.api.views.stat.service') + mock_service = mocker.patch("swh.web.api.views.stat.service") mock_service.stat_counters.side_effect = BadInputExc( - 'voluntary error to check the bad request middleware.') + "voluntary error to check the bad request middleware." + ) - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") rv = api_client.get(url) assert rv.status_code == 400, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'BadInputExc', - 'reason': 'voluntary error to check the bad request middleware.'} + "exception": "BadInputExc", + "reason": "voluntary error to check the bad request middleware.", + } def test_api_1_stat_counters_raise_from_db(api_client, mocker): - mock_service = mocker.patch('swh.web.api.views.stat.service') + mock_service = mocker.patch("swh.web.api.views.stat.service") mock_service.stat_counters.side_effect = StorageDBError( - 'Storage exploded! Will be back online shortly!') + "Storage exploded! Will be back online shortly!" + ) - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") rv = api_client.get(url) assert rv.status_code == 503, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'StorageDBError', - 'reason': - 'An unexpected error occurred in the backend: ' - 'Storage exploded! Will be back online shortly!'} + "exception": "StorageDBError", + "reason": "An unexpected error occurred in the backend: " + "Storage exploded! Will be back online shortly!", + } def test_api_1_stat_counters_raise_from_api(api_client, mocker): - mock_service = mocker.patch('swh.web.api.views.stat.service') + mock_service = mocker.patch("swh.web.api.views.stat.service") mock_service.stat_counters.side_effect = StorageAPIError( - 'Storage API dropped dead! Will resurrect from its ashes asap!' + "Storage API dropped dead! Will resurrect from its ashes asap!" ) - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") rv = api_client.get(url) assert rv.status_code == 503, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == { - 'exception': 'StorageAPIError', - 'reason': - 'An unexpected error occurred in the api backend: ' - 'Storage API dropped dead! Will resurrect from its ashes asap!' + "exception": "StorageAPIError", + "reason": "An unexpected error occurred in the api backend: " + "Storage API dropped dead! Will resurrect from its ashes asap!", } def test_api_1_stat_counters(api_client, archive_data): - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") rv = api_client.get(url) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" assert rv.data == archive_data.stat_counters() diff --git a/swh/web/tests/api/views/test_vault.py b/swh/web/tests/api/views/test_vault.py index 74056d60..d4839ea9 100644 --- a/swh/web/tests/api/views/test_vault.py +++ b/swh/web/tests/api/views/test_vault.py @@ -1,115 +1,130 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.model import hashutil from swh.web.common.utils import reverse -TEST_OBJ_ID = 'd4905454cc154b492bd6afed48694ae3c579345e' +TEST_OBJ_ID = "d4905454cc154b492bd6afed48694ae3c579345e" -OBJECT_TYPES = ('directory', 'revision_gitfast') +OBJECT_TYPES = ("directory", "revision_gitfast") def test_api_vault_cook(api_client, mocker): - mock_service = mocker.patch('swh.web.api.views.vault.service') + mock_service = mocker.patch("swh.web.api.views.vault.service") for obj_type in OBJECT_TYPES: - fetch_url = reverse(f'api-1-vault-fetch-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + fetch_url = reverse( + f"api-1-vault-fetch-{obj_type}", + url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID}, + ) stub_cook = { - 'fetch_url': fetch_url, - 'obj_id': TEST_OBJ_ID, - 'obj_type': obj_type, - 'progress_message': None, - 'status': 'done', - 'task_uuid': 'de75c902-5ee5-4739-996e-448376a93eff', + "fetch_url": fetch_url, + "obj_id": TEST_OBJ_ID, + "obj_type": obj_type, + "progress_message": None, + "status": "done", + "task_uuid": "de75c902-5ee5-4739-996e-448376a93eff", } - stub_fetch = b'content' + stub_fetch = b"content" mock_service.vault_cook.return_value = stub_cook mock_service.vault_fetch.return_value = stub_fetch - url = reverse(f'api-1-vault-cook-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + url = reverse( + f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID} + ) - rv = api_client.post(url, {'email': 'test@test.mail'}) + rv = api_client.post(url, {"email": "test@test.mail"}) assert rv.status_code == 200, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" - stub_cook['fetch_url'] = rv.wsgi_request.build_absolute_uri( - stub_cook['fetch_url']) + stub_cook["fetch_url"] = rv.wsgi_request.build_absolute_uri( + stub_cook["fetch_url"] + ) assert rv.data == stub_cook mock_service.vault_cook.assert_called_with( - obj_type, - hashutil.hash_to_bytes(TEST_OBJ_ID), - 'test@test.mail') + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID), "test@test.mail" + ) rv = api_client.get(fetch_url) assert rv.status_code == 200 - assert rv['Content-Type'] == 'application/gzip' + assert rv["Content-Type"] == "application/gzip" assert rv.content == stub_fetch mock_service.vault_fetch.assert_called_with( - obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID)) + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID) + ) def test_api_vault_cook_uppercase_hash(api_client): for obj_type in OBJECT_TYPES: - url = reverse(f'api-1-vault-cook-{obj_type}-uppercase-checksum', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID.upper()}) - rv = api_client.post(url, {'email': 'test@test.mail'}) + url = reverse( + f"api-1-vault-cook-{obj_type}-uppercase-checksum", + url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID.upper()}, + ) + rv = api_client.post(url, {"email": "test@test.mail"}) assert rv.status_code == 302 - redirect_url = reverse(f'api-1-vault-cook-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + redirect_url = reverse( + f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID} + ) - assert rv['location'] == redirect_url + assert rv["location"] == redirect_url fetch_url = reverse( - f'api-1-vault-fetch-{obj_type}-uppercase-checksum', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID.upper()}) + f"api-1-vault-fetch-{obj_type}-uppercase-checksum", + url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID.upper()}, + ) rv = api_client.get(fetch_url) assert rv.status_code == 302 - redirect_url = reverse(f'api-1-vault-fetch-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + redirect_url = reverse( + f"api-1-vault-fetch-{obj_type}", + url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID}, + ) - assert rv['location'] == redirect_url + assert rv["location"] == redirect_url def test_api_vault_cook_notfound(api_client, mocker): - mock_service = mocker.patch('swh.web.api.views.vault.service') + mock_service = mocker.patch("swh.web.api.views.vault.service") mock_service.vault_cook.return_value = None mock_service.vault_fetch.return_value = None for obj_type in OBJECT_TYPES: - url = reverse(f'api-1-vault-cook-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + url = reverse( + f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID} + ) rv = api_client.post(url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' + assert rv["Content-Type"] == "application/json" - assert rv.data['exception'] == 'NotFoundExc' + assert rv.data["exception"] == "NotFoundExc" mock_service.vault_cook.assert_called_with( - obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID), None) + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID), None + ) - fetch_url = reverse(f'api-1-vault-fetch-{obj_type}', - url_args={f'{obj_type[:3]}_id': TEST_OBJ_ID}) + fetch_url = reverse( + f"api-1-vault-fetch-{obj_type}", + url_args={f"{obj_type[:3]}_id": TEST_OBJ_ID}, + ) rv = api_client.get(fetch_url) assert rv.status_code == 404, rv.data - assert rv['Content-Type'] == 'application/json' - assert rv.data['exception'] == 'NotFoundExc' + assert rv["Content-Type"] == "application/json" + assert rv.data["exception"] == "NotFoundExc" mock_service.vault_fetch.assert_called_with( - obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID)) + obj_type, hashutil.hash_to_bytes(TEST_OBJ_ID) + ) diff --git a/swh/web/tests/auth/keycloak_mock.py b/swh/web/tests/auth/keycloak_mock.py index a6cdf3df..3cbe9448 100644 --- a/swh/web/tests/auth/keycloak_mock.py +++ b/swh/web/tests/auth/keycloak_mock.py @@ -1,86 +1,98 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from copy import copy from unittest.mock import Mock from django.utils import timezone from swh.web.auth.keycloak import KeycloakOpenIDConnect from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID from swh.web.config import get_config from .sample_data import oidc_profile, realm_public_key, userinfo class KeycloackOpenIDConnectMock(KeycloakOpenIDConnect): - def __init__(self, auth_success=True, exp=None): swhweb_config = get_config() - super().__init__(swhweb_config['keycloak']['server_url'], - swhweb_config['keycloak']['realm_name'], - OIDC_SWH_WEB_CLIENT_ID) + super().__init__( + swhweb_config["keycloak"]["server_url"], + swhweb_config["keycloak"]["realm_name"], + OIDC_SWH_WEB_CLIENT_ID, + ) self.auth_success = auth_success self.exp = exp self._keycloak.public_key = lambda: realm_public_key self._keycloak.well_know = lambda: { - 'issuer': f'{self.server_url}realms/{self.realm_name}', - 'authorization_endpoint': (f'{self.server_url}realms/' - f'{self.realm_name}/protocol/' - 'openid-connect/auth'), - 'token_endpoint': (f'{self.server_url}realms/{self.realm_name}/' - 'protocol/openid-connect/token'), - 'token_introspection_endpoint': (f'{self.server_url}realms/' - f'{self.realm_name}/protocol/' - 'openid-connect/token/' - 'introspect'), - 'userinfo_endpoint': (f'{self.server_url}realms/{self.realm_name}/' - 'protocol/openid-connect/userinfo'), - 'end_session_endpoint': (f'{self.server_url}realms/' - f'{self.realm_name}/protocol/' - 'openid-connect/logout'), - 'jwks_uri': (f'{self.server_url}realms/{self.realm_name}/' - 'protocol/openid-connect/certs'), + "issuer": f"{self.server_url}realms/{self.realm_name}", + "authorization_endpoint": ( + f"{self.server_url}realms/" + f"{self.realm_name}/protocol/" + "openid-connect/auth" + ), + "token_endpoint": ( + f"{self.server_url}realms/{self.realm_name}/" + "protocol/openid-connect/token" + ), + "token_introspection_endpoint": ( + f"{self.server_url}realms/" + f"{self.realm_name}/protocol/" + "openid-connect/token/" + "introspect" + ), + "userinfo_endpoint": ( + f"{self.server_url}realms/{self.realm_name}/" + "protocol/openid-connect/userinfo" + ), + "end_session_endpoint": ( + f"{self.server_url}realms/" + f"{self.realm_name}/protocol/" + "openid-connect/logout" + ), + "jwks_uri": ( + f"{self.server_url}realms/{self.realm_name}/" + "protocol/openid-connect/certs" + ), } self.authorization_code = Mock() self.userinfo = Mock() self.logout = Mock() if auth_success: self.authorization_code.return_value = copy(oidc_profile) self.userinfo.return_value = copy(userinfo) else: self.authorization_url = Mock() - exception = Exception('Authentication failed') + exception = Exception("Authentication failed") self.authorization_code.side_effect = exception self.authorization_url.side_effect = exception self.userinfo.side_effect = exception self.logout.side_effect = exception def decode_token(self, token): options = {} if self.auth_success: # skip signature expiration check as we use a static oidc_profile # for the tests with expired tokens in it - options['verify_exp'] = False + options["verify_exp"] = False decoded = super().decode_token(token, options) # tweak auth and exp time for tests - expire_in = decoded['exp'] - decoded['auth_time'] + expire_in = decoded["exp"] - decoded["auth_time"] if self.exp is not None: - decoded['exp'] = self.exp - decoded['auth_time'] = self.exp - expire_in + decoded["exp"] = self.exp + decoded["auth_time"] = self.exp - expire_in else: - decoded['auth_time'] = int(timezone.now().timestamp()) - decoded['exp'] = decoded['auth_time'] + expire_in - decoded['groups'] = ['/staff'] + decoded["auth_time"] = int(timezone.now().timestamp()) + decoded["exp"] = decoded["auth_time"] + expire_in + decoded["groups"] = ["/staff"] return decoded def mock_keycloak(mocker, auth_success=True, exp=None): kc_oidc_mock = KeycloackOpenIDConnectMock(auth_success, exp) - mock_get_oidc_client = mocker.patch( - 'swh.web.auth.views.get_oidc_client') + mock_get_oidc_client = mocker.patch("swh.web.auth.views.get_oidc_client") mock_get_oidc_client.return_value = kc_oidc_mock - mocker.patch('swh.web.auth.backends._oidc_client', kc_oidc_mock) + mocker.patch("swh.web.auth.backends._oidc_client", kc_oidc_mock) return kc_oidc_mock diff --git a/swh/web/tests/auth/sample_data.py b/swh/web/tests/auth/sample_data.py index 604bae28..02cd843e 100644 --- a/swh/web/tests/auth/sample_data.py +++ b/swh/web/tests/auth/sample_data.py @@ -1,95 +1,101 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information realm_public_key = ( - 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnqF4xvGjaI54P6WtJvyGayxP8A93u' - 'NcA3TH6jitwmyAalj8dN8/NzK9vrdlSA3Ibvp/XQujPSOP7a35YiYFscEJnogTXQpE/FhZrUY' - 'y21U6ezruVUv4z/ER1cYLb+q5ZI86nXSTNCAbH+lw7rQjlvcJ9KvgHEeA5ALXJ1r55zUmNvuy' - '5o6ke1G3fXbNSXwF4qlWAzo1o7Ms8qNrNyOG8FPx24dvm9xMH7/08IPvh9KUqlnP8h6olpxHr' - 'drX/q4E+Nzj8Tr8p7Z5CimInls40QuOTIhs6C2SwFHUgQgXl9hB9umiZJlwYEpDv0/LO2zYie' - 'Hl5Lv7Iig4FOIXIVCaDGQIDAQAB' + "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnqF4xvGjaI54P6WtJvyGayxP8A93u" + "NcA3TH6jitwmyAalj8dN8/NzK9vrdlSA3Ibvp/XQujPSOP7a35YiYFscEJnogTXQpE/FhZrUY" + "y21U6ezruVUv4z/ER1cYLb+q5ZI86nXSTNCAbH+lw7rQjlvcJ9KvgHEeA5ALXJ1r55zUmNvuy" + "5o6ke1G3fXbNSXwF4qlWAzo1o7Ms8qNrNyOG8FPx24dvm9xMH7/08IPvh9KUqlnP8h6olpxHr" + "drX/q4E+Nzj8Tr8p7Z5CimInls40QuOTIhs6C2SwFHUgQgXl9hB9umiZJlwYEpDv0/LO2zYie" + "Hl5Lv7Iig4FOIXIVCaDGQIDAQAB" ) oidc_profile = { - 'access_token': ('eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPSnhV' - 'Q0p0TmJQT0NOUGFNNmc3ZU1zY2pqTXhoem9vNGxZaFhsa1c2TWhBIn0.' - 'eyJqdGkiOiIzMWZjNTBiNy1iYmU1LTRmNTEtOTFlZi04ZTNlZWM1MTMz' - 'MWUiLCJleHAiOjE1ODI3MjM3MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIz' - 'MTAxLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFs' - 'bXMvU29mdHdhcmVIZXJpdGFnZSIsImF1ZCI6WyJzd2gtd2ViIiwiYWNj' - 'b3VudCJdLCJzdWIiOiJmZWFjZDM0NC1iNDY4LTRhNjUtYTIzNi0xNGY2' - 'MWU2YjcyMDAiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJzd2gtd2ViIiwi' - 'YXV0aF90aW1lIjoxNTgyNzIzMTAwLCJzZXNzaW9uX3N0YXRlIjoiZDgy' - 'YjkwZDEtMGE5NC00ZTc0LWFkNjYtZGQ5NTM0MWM3YjZkIiwiYWNyIjoi' - 'MSIsImFsbG93ZWQtb3JpZ2lucyI6WyIqIl0sInJlYWxtX2FjY2VzcyI6' - 'eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0' - 'aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xl' - 'cyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtz' - 'Iiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgZW1haWwg' - 'cHJvZmlsZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6Ikpv' - 'aG4gRG9lIiwiZ3JvdXBzIjpbXSwicHJlZmVycmVkX3VzZXJuYW1lIjoi' - 'am9obmRvZSIsImdpdmVuX25hbWUiOiJKb2huIiwiZmFtaWx5X25hbWUi' - 'OiJEb2UiLCJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIn0.neJ-' - 'Pmd87J6Gt0fzDqmXFeoy34Iqb5vNNEEgIKqtqg3moaVkbXrO_9R37DJB' - 'AgdFv0owVONK3GbqPOEICePgG6RFtri999DetNE-O5sB4fwmHPWcHPlO' - 'kcPLbVJqu6zWo-2AzlfAy5bCNvj_wzs2tjFjLeHcRgR1a1WY3uTp5EWc' - 'HITCWQZzZWFGZTZCTlGkpdyJTqxGBdSHRB4NlIVGpYSTBsBsxttFEetl' - 'rpcNd4-5AteFprIr9hn9VasIIF8WdFdtC2e8xGMJW5Q0M3G3Iu-LLNmE' - 'oTIDqtbJ7OrIcGBIwsc3seCV3eCG6kOYwz5w-f8DeOpwcDX58yYPmapJ' - '6A'), - 'expires_in': 600, - 'id_token': ('eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPSnhVQ0p0' - 'TmJQT0NOUGFNNmc3ZU1zY2pqTXhoem9vNGxZaFhsa1c2TWhBIn0.eyJqdGki' - 'OiI0NDRlYzU1My1iYzhiLTQ2YjYtOTlmYS0zOTc3YTJhZDY1ZmEiLCJleHAi' - 'OjE1ODI3MjM3MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIzMTAxLCJpc3MiOiJo' - 'dHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFsbXMvU29mdHdhcmVIZXJp' - 'dGFnZSIsImF1ZCI6InN3aC13ZWIiLCJzdWIiOiJmZWFjZDM0NC1iNDY4LTRh' - 'NjUtYTIzNi0xNGY2MWU2YjcyMDAiLCJ0eXAiOiJJRCIsImF6cCI6InN3aC13' - 'ZWIiLCJhdXRoX3RpbWUiOjE1ODI3MjMxMDAsInNlc3Npb25fc3RhdGUiOiJk' - 'ODJiOTBkMS0wYTk0LTRlNzQtYWQ2Ni1kZDk1MzQxYzdiNmQiLCJhY3IiOiIx' - 'IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoiSm9obiBEb2UiLCJn' - 'cm91cHMiOltdLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJqb2huZG9lIiwiZ2l2' - 'ZW5fbmFtZSI6IkpvaG4iLCJmYW1pbHlfbmFtZSI6IkRvZSIsImVtYWlsIjoi' - 'am9obi5kb2VAZXhhbXBsZS5jb20ifQ.YB7bxlz_wgLJSkylVjmqedxQgEMee' - 'JOdi9CFHXV4F3ZWsEZ52CGuJXsozkX2oXvgU06MzzLNEK8ojgrPSNzjRkutL' - 'aaLq_YUzv4iV8fmKUS_aEyiYZbfoBe3Y4dwv2FoPEPCt96iTwpzM5fg_oYw_' - 'PHCq-Yl5SulT1nTrJZpntkf0hRjmxlDO06JMp0aZ8xS8RYJqH48xCRf_DARE' - '0jJV2-UuzOWI6xBATwFfP44kV6wFmErLN5txMgwZzCSB2OCe5Cl1il0eTQTN' - 'ybeSYZeZE61QtuTRUHeP1D1qSbJGy5g_S67SdTkS-hQFvfrrD84qGflIEqnX' - 'ZbYnitD1Typ6Q'), - 'not-before-policy': 0, - 'refresh_expires_in': 1800, - 'refresh_token': ('eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJmNjM' - 'zMDE5MS01YTU4LTQxMDAtOGIzYS00ZDdlM2U1NjA3MTgifQ.eyJqdGk' - 'iOiIxYWI5ZWZmMS0xZWZlLTQ3MDMtOGQ2YS03Nzg1NWUwYzQyYTYiLC' - 'JleHAiOjE1ODI3MjQ5MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIzMTAxL' - 'CJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFsbXMv' - 'U29mdHdhcmVIZXJpdGFnZSIsImF1ZCI6Imh0dHA6Ly9sb2NhbGhvc3Q' - '6ODA4MC9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwic3ViIj' - 'oiZmVhY2QzNDQtYjQ2OC00YTY1LWEyMzYtMTRmNjFlNmI3MjAwIiwid' - 'HlwIjoiUmVmcmVzaCIsImF6cCI6InN3aC13ZWIiLCJhdXRoX3RpbWUi' - 'OjAsInNlc3Npb25fc3RhdGUiOiJkODJiOTBkMS0wYTk0LTRlNzQtYWQ' - '2Ni1kZDk1MzQxYzdiNmQiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOl' - 'sib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwic' - 'mVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFu' - 'YWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXc' - 'tcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbG' - 'UifQ.xQYrl2CMP_GQ_TFqhsTz-rTs3WuZz5I37toi1eSsDMI'), - 'scope': 'openid email profile', - 'session_state': 'd82b90d1-0a94-4e74-ad66-dd95341c7b6d', - 'token_type': 'bearer' + "access_token": ( + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPSnhV" + "Q0p0TmJQT0NOUGFNNmc3ZU1zY2pqTXhoem9vNGxZaFhsa1c2TWhBIn0." + "eyJqdGkiOiIzMWZjNTBiNy1iYmU1LTRmNTEtOTFlZi04ZTNlZWM1MTMz" + "MWUiLCJleHAiOjE1ODI3MjM3MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIz" + "MTAxLCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFs" + "bXMvU29mdHdhcmVIZXJpdGFnZSIsImF1ZCI6WyJzd2gtd2ViIiwiYWNj" + "b3VudCJdLCJzdWIiOiJmZWFjZDM0NC1iNDY4LTRhNjUtYTIzNi0xNGY2" + "MWU2YjcyMDAiLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiJzd2gtd2ViIiwi" + "YXV0aF90aW1lIjoxNTgyNzIzMTAwLCJzZXNzaW9uX3N0YXRlIjoiZDgy" + "YjkwZDEtMGE5NC00ZTc0LWFkNjYtZGQ5NTM0MWM3YjZkIiwiYWNyIjoi" + "MSIsImFsbG93ZWQtb3JpZ2lucyI6WyIqIl0sInJlYWxtX2FjY2VzcyI6" + "eyJyb2xlcyI6WyJvZmZsaW5lX2FjY2VzcyIsInVtYV9hdXRob3JpemF0" + "aW9uIl19LCJyZXNvdXJjZV9hY2Nlc3MiOnsiYWNjb3VudCI6eyJyb2xl" + "cyI6WyJtYW5hZ2UtYWNjb3VudCIsIm1hbmFnZS1hY2NvdW50LWxpbmtz" + "Iiwidmlldy1wcm9maWxlIl19fSwic2NvcGUiOiJvcGVuaWQgZW1haWwg" + "cHJvZmlsZSIsImVtYWlsX3ZlcmlmaWVkIjpmYWxzZSwibmFtZSI6Ikpv" + "aG4gRG9lIiwiZ3JvdXBzIjpbXSwicHJlZmVycmVkX3VzZXJuYW1lIjoi" + "am9obmRvZSIsImdpdmVuX25hbWUiOiJKb2huIiwiZmFtaWx5X25hbWUi" + "OiJEb2UiLCJlbWFpbCI6ImpvaG4uZG9lQGV4YW1wbGUuY29tIn0.neJ-" + "Pmd87J6Gt0fzDqmXFeoy34Iqb5vNNEEgIKqtqg3moaVkbXrO_9R37DJB" + "AgdFv0owVONK3GbqPOEICePgG6RFtri999DetNE-O5sB4fwmHPWcHPlO" + "kcPLbVJqu6zWo-2AzlfAy5bCNvj_wzs2tjFjLeHcRgR1a1WY3uTp5EWc" + "HITCWQZzZWFGZTZCTlGkpdyJTqxGBdSHRB4NlIVGpYSTBsBsxttFEetl" + "rpcNd4-5AteFprIr9hn9VasIIF8WdFdtC2e8xGMJW5Q0M3G3Iu-LLNmE" + "oTIDqtbJ7OrIcGBIwsc3seCV3eCG6kOYwz5w-f8DeOpwcDX58yYPmapJ" + "6A" + ), + "expires_in": 600, + "id_token": ( + "eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJPSnhVQ0p0" + "TmJQT0NOUGFNNmc3ZU1zY2pqTXhoem9vNGxZaFhsa1c2TWhBIn0.eyJqdGki" + "OiI0NDRlYzU1My1iYzhiLTQ2YjYtOTlmYS0zOTc3YTJhZDY1ZmEiLCJleHAi" + "OjE1ODI3MjM3MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIzMTAxLCJpc3MiOiJo" + "dHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFsbXMvU29mdHdhcmVIZXJp" + "dGFnZSIsImF1ZCI6InN3aC13ZWIiLCJzdWIiOiJmZWFjZDM0NC1iNDY4LTRh" + "NjUtYTIzNi0xNGY2MWU2YjcyMDAiLCJ0eXAiOiJJRCIsImF6cCI6InN3aC13" + "ZWIiLCJhdXRoX3RpbWUiOjE1ODI3MjMxMDAsInNlc3Npb25fc3RhdGUiOiJk" + "ODJiOTBkMS0wYTk0LTRlNzQtYWQ2Ni1kZDk1MzQxYzdiNmQiLCJhY3IiOiIx" + "IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJuYW1lIjoiSm9obiBEb2UiLCJn" + "cm91cHMiOltdLCJwcmVmZXJyZWRfdXNlcm5hbWUiOiJqb2huZG9lIiwiZ2l2" + "ZW5fbmFtZSI6IkpvaG4iLCJmYW1pbHlfbmFtZSI6IkRvZSIsImVtYWlsIjoi" + "am9obi5kb2VAZXhhbXBsZS5jb20ifQ.YB7bxlz_wgLJSkylVjmqedxQgEMee" + "JOdi9CFHXV4F3ZWsEZ52CGuJXsozkX2oXvgU06MzzLNEK8ojgrPSNzjRkutL" + "aaLq_YUzv4iV8fmKUS_aEyiYZbfoBe3Y4dwv2FoPEPCt96iTwpzM5fg_oYw_" + "PHCq-Yl5SulT1nTrJZpntkf0hRjmxlDO06JMp0aZ8xS8RYJqH48xCRf_DARE" + "0jJV2-UuzOWI6xBATwFfP44kV6wFmErLN5txMgwZzCSB2OCe5Cl1il0eTQTN" + "ybeSYZeZE61QtuTRUHeP1D1qSbJGy5g_S67SdTkS-hQFvfrrD84qGflIEqnX" + "ZbYnitD1Typ6Q" + ), + "not-before-policy": 0, + "refresh_expires_in": 1800, + "refresh_token": ( + "eyJhbGciOiJIUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJmNjM" + "zMDE5MS01YTU4LTQxMDAtOGIzYS00ZDdlM2U1NjA3MTgifQ.eyJqdGk" + "iOiIxYWI5ZWZmMS0xZWZlLTQ3MDMtOGQ2YS03Nzg1NWUwYzQyYTYiLC" + "JleHAiOjE1ODI3MjQ5MDEsIm5iZiI6MCwiaWF0IjoxNTgyNzIzMTAxL" + "CJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjgwODAvYXV0aC9yZWFsbXMv" + "U29mdHdhcmVIZXJpdGFnZSIsImF1ZCI6Imh0dHA6Ly9sb2NhbGhvc3Q" + "6ODA4MC9hdXRoL3JlYWxtcy9Tb2Z0d2FyZUhlcml0YWdlIiwic3ViIj" + "oiZmVhY2QzNDQtYjQ2OC00YTY1LWEyMzYtMTRmNjFlNmI3MjAwIiwid" + "HlwIjoiUmVmcmVzaCIsImF6cCI6InN3aC13ZWIiLCJhdXRoX3RpbWUi" + "OjAsInNlc3Npb25fc3RhdGUiOiJkODJiOTBkMS0wYTk0LTRlNzQtYWQ" + "2Ni1kZDk1MzQxYzdiNmQiLCJyZWFsbV9hY2Nlc3MiOnsicm9sZXMiOl" + "sib2ZmbGluZV9hY2Nlc3MiLCJ1bWFfYXV0aG9yaXphdGlvbiJdfSwic" + "mVzb3VyY2VfYWNjZXNzIjp7ImFjY291bnQiOnsicm9sZXMiOlsibWFu" + "YWdlLWFjY291bnQiLCJtYW5hZ2UtYWNjb3VudC1saW5rcyIsInZpZXc" + "tcHJvZmlsZSJdfX0sInNjb3BlIjoib3BlbmlkIGVtYWlsIHByb2ZpbG" + "UifQ.xQYrl2CMP_GQ_TFqhsTz-rTs3WuZz5I37toi1eSsDMI" + ), + "scope": "openid email profile", + "session_state": "d82b90d1-0a94-4e74-ad66-dd95341c7b6d", + "token_type": "bearer", } userinfo = { - 'email': 'john.doe@example.com', - 'email_verified': False, - 'family_name': 'Doe', - 'given_name': 'John', - 'groups': ['/staff'], - 'name': 'John Doe', - 'preferred_username': 'johndoe', - 'sub': 'feacd344-b468-4a65-a236-14f61e6b7200' + "email": "john.doe@example.com", + "email_verified": False, + "family_name": "Doe", + "given_name": "John", + "groups": ["/staff"], + "name": "John Doe", + "preferred_username": "johndoe", + "sub": "feacd344-b468-4a65-a236-14f61e6b7200", } diff --git a/swh/web/tests/auth/test_api_auth.py b/swh/web/tests/auth/test_api_auth.py index 9cdd808b..b35a4885 100644 --- a/swh/web/tests/auth/test_api_auth.py +++ b/swh/web/tests/auth/test_api_auth.py @@ -1,120 +1,115 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from django.contrib.auth.models import AnonymousUser, User from swh.web.auth.models import OIDCUser from swh.web.common.utils import reverse from .keycloak_mock import mock_keycloak from . import sample_data @pytest.mark.django_db def test_drf_django_session_auth_success(mocker, client): """ Check user gets authenticated when querying the web api through a web browser. """ - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") mock_keycloak(mocker) - client.login(code='', code_verifier='', redirect_uri='') + client.login(code="", code_verifier="", redirect_uri="") response = client.get(url) request = response.wsgi_request assert response.status_code == 200 # user should be authenticated assert isinstance(request.user, OIDCUser) # check remoter used has not been saved to Django database with pytest.raises(User.DoesNotExist): User.objects.get(username=request.user.username) @pytest.mark.django_db def test_drf_oidc_bearer_token_auth_success(mocker, api_client): """ Check user gets authenticated when querying the web api through an HTTP client using bearer token authentication. """ - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] mock_keycloak(mocker) - api_client.credentials( - HTTP_AUTHORIZATION=f"Bearer {access_token}") + api_client.credentials(HTTP_AUTHORIZATION=f"Bearer {access_token}") response = api_client.get(url) request = response.wsgi_request assert response.status_code == 200 # user should be authenticated assert isinstance(request.user, OIDCUser) # check remoter used has not been saved to Django database with pytest.raises(User.DoesNotExist): User.objects.get(username=request.user.username) @pytest.mark.django_db def test_drf_oidc_bearer_token_auth_failure(mocker, api_client): - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] # check for failed authentication but with expected token format mock_keycloak(mocker, auth_success=False) - api_client.credentials( - HTTP_AUTHORIZATION=f"Bearer {access_token}") + api_client.credentials(HTTP_AUTHORIZATION=f"Bearer {access_token}") response = api_client.get(url) request = response.wsgi_request assert response.status_code == 403 assert isinstance(request.user, AnonymousUser) # check for failed authentication when token format is invalid mock_keycloak(mocker) - api_client.credentials( - HTTP_AUTHORIZATION=f"Bearer invalid-token-format") + api_client.credentials(HTTP_AUTHORIZATION=f"Bearer invalid-token-format") response = api_client.get(url) request = response.wsgi_request assert response.status_code == 403 assert isinstance(request.user, AnonymousUser) def test_drf_oidc_auth_invalid_or_missing_authorization_type(api_client): - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] # missing authorization type - api_client.credentials( - HTTP_AUTHORIZATION=f"{access_token}") + api_client.credentials(HTTP_AUTHORIZATION=f"{access_token}") response = api_client.get(url) request = response.wsgi_request assert response.status_code == 403 assert isinstance(request.user, AnonymousUser) # invalid authorization type - api_client.credentials( - HTTP_AUTHORIZATION=f"Foo token") + api_client.credentials(HTTP_AUTHORIZATION=f"Foo token") response = api_client.get(url) request = response.wsgi_request assert response.status_code == 403 assert isinstance(request.user, AnonymousUser) diff --git a/swh/web/tests/auth/test_backends.py b/swh/web/tests/auth/test_backends.py index 58cc6cb8..9ab6738f 100644 --- a/swh/web/tests/auth/test_backends.py +++ b/swh/web/tests/auth/test_backends.py @@ -1,150 +1,148 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timedelta from django.contrib.auth import authenticate, get_backends import pytest from django.conf import settings from rest_framework.exceptions import AuthenticationFailed from swh.web.auth.backends import OIDCBearerTokenAuthentication from swh.web.auth.models import OIDCUser from swh.web.common.utils import reverse from . import sample_data from .keycloak_mock import mock_keycloak def _authenticate_user(request_factory): - request = request_factory.get(reverse('oidc-login-complete')) + request = request_factory.get(reverse("oidc-login-complete")) - return authenticate(request=request, - code='some-code', - code_verifier='some-code-verifier', - redirect_uri='https://localhost:5004') + return authenticate( + request=request, + code="some-code", + code_verifier="some-code-verifier", + redirect_uri="https://localhost:5004", + ) def _check_authenticated_user(user, decoded_token): assert user is not None assert isinstance(user, OIDCUser) assert user.id != 0 - assert user.username == decoded_token['preferred_username'] - assert user.password == '' - assert user.first_name == decoded_token['given_name'] - assert user.last_name == decoded_token['family_name'] - assert user.email == decoded_token['email'] - assert user.is_staff == ('/staff' in decoded_token['groups']) - assert user.sub == decoded_token['sub'] + assert user.username == decoded_token["preferred_username"] + assert user.password == "" + assert user.first_name == decoded_token["given_name"] + assert user.last_name == decoded_token["family_name"] + assert user.email == decoded_token["email"] + assert user.is_staff == ("/staff" in decoded_token["groups"]) + assert user.sub == decoded_token["sub"] @pytest.mark.django_db def test_oidc_code_pkce_auth_backend_success(mocker, request_factory): kc_oidc_mock = mock_keycloak(mocker) oidc_profile = sample_data.oidc_profile user = _authenticate_user(request_factory) decoded_token = kc_oidc_mock.decode_token(user.access_token) _check_authenticated_user(user, decoded_token) - auth_datetime = datetime.fromtimestamp(decoded_token['auth_time']) - exp_datetime = datetime.fromtimestamp(decoded_token['exp']) - refresh_exp_datetime = ( - auth_datetime + timedelta(seconds=oidc_profile['refresh_expires_in'])) + auth_datetime = datetime.fromtimestamp(decoded_token["auth_time"]) + exp_datetime = datetime.fromtimestamp(decoded_token["exp"]) + refresh_exp_datetime = auth_datetime + timedelta( + seconds=oidc_profile["refresh_expires_in"] + ) - assert user.access_token == oidc_profile['access_token'] + assert user.access_token == oidc_profile["access_token"] assert user.expires_at == exp_datetime - assert user.id_token == oidc_profile['id_token'] - assert user.refresh_token == oidc_profile['refresh_token'] + assert user.id_token == oidc_profile["id_token"] + assert user.refresh_token == oidc_profile["refresh_token"] assert user.refresh_expires_at == refresh_exp_datetime - assert user.scope == oidc_profile['scope'] - assert user.session_state == oidc_profile['session_state'] + assert user.scope == oidc_profile["scope"] + assert user.session_state == oidc_profile["session_state"] - backend_path = 'swh.web.auth.backends.OIDCAuthorizationCodePKCEBackend' + backend_path = "swh.web.auth.backends.OIDCAuthorizationCodePKCEBackend" assert user.backend == backend_path backend_idx = settings.AUTHENTICATION_BACKENDS.index(backend_path) assert get_backends()[backend_idx].get_user(user.id) == user @pytest.mark.django_db def test_oidc_code_pkce_auth_backend_failure(mocker, request_factory): mock_keycloak(mocker, auth_success=False) user = _authenticate_user(request_factory) assert user is None @pytest.mark.django_db -def test_drf_oidc_bearer_token_auth_backend_success(mocker, - api_request_factory): - url = reverse('api-1-stat-counters') +def test_drf_oidc_bearer_token_auth_backend_success(mocker, api_request_factory): + url = reverse("api-1-stat-counters") drf_auth_backend = OIDCBearerTokenAuthentication() kc_oidc_mock = mock_keycloak(mocker) - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] decoded_token = kc_oidc_mock.decode_token(access_token) - request = api_request_factory.get( - url, HTTP_AUTHORIZATION=f"Bearer {access_token}") + request = api_request_factory.get(url, HTTP_AUTHORIZATION=f"Bearer {access_token}") user, _ = drf_auth_backend.authenticate(request) _check_authenticated_user(user, decoded_token) # oidc_profile is not filled when authenticating through bearer token - assert hasattr(user, 'access_token') and user.access_token is None + assert hasattr(user, "access_token") and user.access_token is None @pytest.mark.django_db -def test_drf_oidc_bearer_token_auth_backend_failure(mocker, - api_request_factory): +def test_drf_oidc_bearer_token_auth_backend_failure(mocker, api_request_factory): - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") drf_auth_backend = OIDCBearerTokenAuthentication() # simulate a failed authentication with a bearer token in expected format mock_keycloak(mocker, auth_success=False) - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] - request = api_request_factory.get( - url, HTTP_AUTHORIZATION=f"Bearer {access_token}") + request = api_request_factory.get(url, HTTP_AUTHORIZATION=f"Bearer {access_token}") with pytest.raises(AuthenticationFailed): drf_auth_backend.authenticate(request) # simulate a failed authentication with an invalid bearer token format mock_keycloak(mocker) request = api_request_factory.get( - url, HTTP_AUTHORIZATION=f"Bearer invalid-token-format") + url, HTTP_AUTHORIZATION=f"Bearer invalid-token-format" + ) with pytest.raises(AuthenticationFailed): drf_auth_backend.authenticate(request) def test_drf_oidc_auth_invalid_or_missing_auth_type(api_request_factory): - url = reverse('api-1-stat-counters') + url = reverse("api-1-stat-counters") drf_auth_backend = OIDCBearerTokenAuthentication() - access_token = sample_data.oidc_profile['access_token'] + access_token = sample_data.oidc_profile["access_token"] # Invalid authorization type - request = api_request_factory.get( - url, HTTP_AUTHORIZATION=f"Foo token") + request = api_request_factory.get(url, HTTP_AUTHORIZATION=f"Foo token") with pytest.raises(AuthenticationFailed): drf_auth_backend.authenticate(request) # Missing authorization type - request = api_request_factory.get( - url, HTTP_AUTHORIZATION=f"{access_token}") + request = api_request_factory.get(url, HTTP_AUTHORIZATION=f"{access_token}") with pytest.raises(AuthenticationFailed): drf_auth_backend.authenticate(request) diff --git a/swh/web/tests/auth/test_middlewares.py b/swh/web/tests/auth/test_middlewares.py index 93c06935..2612bd6e 100644 --- a/swh/web/tests/auth/test_middlewares.py +++ b/swh/web/tests/auth/test_middlewares.py @@ -1,46 +1,47 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from django.test import modify_settings import pytest from .keycloak_mock import mock_keycloak from swh.web.common.utils import reverse @pytest.mark.django_db -@modify_settings(MIDDLEWARE={ - 'remove': ['swh.web.auth.middlewares.OIDCSessionRefreshMiddleware'] -}) +@modify_settings( + MIDDLEWARE={"remove": ["swh.web.auth.middlewares.OIDCSessionRefreshMiddleware"]} +) def test_oidc_session_refresh_middleware_disabled(client, mocker): # authenticate but make session expires immediately kc_oidc_mock = mock_keycloak(mocker, exp=int(datetime.now().timestamp())) - client.login(code='', code_verifier='', redirect_uri='') + client.login(code="", code_verifier="", redirect_uri="") kc_oidc_mock.authorization_code.assert_called() - url = reverse('swh-web-homepage') + url = reverse("swh-web-homepage") resp = client.get(url) # no redirection for silent refresh assert resp.status_code != 302 @pytest.mark.django_db def test_oidc_session_refresh_middleware_enabled(client, mocker): # authenticate but make session expires immediately kc_oidc_mock = mock_keycloak(mocker, exp=int(datetime.now().timestamp())) - client.login(code='', code_verifier='', redirect_uri='') + client.login(code="", code_verifier="", redirect_uri="") kc_oidc_mock.authorization_code.assert_called() - url = reverse('swh-web-homepage') + url = reverse("swh-web-homepage") resp = client.get(url) # should redirect for silent session refresh assert resp.status_code == 302 - silent_refresh_url = reverse('oidc-login', query_params={'next_path': url, - 'prompt': 'none'}) - assert resp['location'] == silent_refresh_url + silent_refresh_url = reverse( + "oidc-login", query_params={"next_path": url, "prompt": "none"} + ) + assert resp["location"] == silent_refresh_url diff --git a/swh/web/tests/auth/test_utils.py b/swh/web/tests/auth/test_utils.py index d564b199..9b4c0e0d 100644 --- a/swh/web/tests/auth/test_utils.py +++ b/swh/web/tests/auth/test_utils.py @@ -1,37 +1,37 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import hashlib import re from base64 import urlsafe_b64encode from swh.web.auth.utils import gen_oidc_pkce_codes def test_gen_oidc_pkce_codes(): """ Check generated PKCE codes respect the specification (see https://tools.ietf.org/html/rfc7636#section-4.1) """ code_verifier, code_challenge = gen_oidc_pkce_codes() # check the code verifier only contains allowed characters - assert re.match(r'[a-zA-Z0-9-\._~]+', code_verifier) + assert re.match(r"[a-zA-Z0-9-\._~]+", code_verifier) # check minimum and maximum authorized length for the # code verifier assert len(code_verifier) >= 43 assert len(code_verifier) <= 128 # compute code challenge from code verifier - challenge = hashlib.sha256(code_verifier.encode('ascii')).digest() - challenge = urlsafe_b64encode(challenge).decode('ascii') - challenge = challenge.replace('=', '') + challenge = hashlib.sha256(code_verifier.encode("ascii")).digest() + challenge = urlsafe_b64encode(challenge).decode("ascii") + challenge = challenge.replace("=", "") # check base64 padding is not present - assert not code_challenge[-1].endswith('=') + assert not code_challenge[-1].endswith("=") # check code challenge is valid assert code_challenge == challenge diff --git a/swh/web/tests/auth/test_views.py b/swh/web/tests/auth/test_views.py index e344dceb..0a5eb38c 100644 --- a/swh/web/tests/auth/test_views.py +++ b/swh/web/tests/auth/test_views.py @@ -1,316 +1,328 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from urllib.parse import urljoin, urlparse import uuid from django.http import QueryDict from django.contrib.auth.models import AnonymousUser, User import pytest from swh.web.auth.models import OIDCUser from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_template_used, assert_contains from . import sample_data from .keycloak_mock import mock_keycloak @pytest.mark.django_db def test_oidc_login_views_success(client, mocker): """ Simulate a successful login authentication with OpenID Connect authorization code flow with PKCE. """ # mock Keycloak client kc_oidc_mock = mock_keycloak(mocker) # user initiates login process - login_url = reverse('oidc-login') + login_url = reverse("oidc-login") response = client.get(login_url) request = response.wsgi_request # should redirect to Keycloak authentication page in order # for a user to login with its username / password assert response.status_code == 302 assert isinstance(request.user, AnonymousUser) - parsed_url = urlparse(response['location']) + parsed_url = urlparse(response["location"]) - authorization_url = kc_oidc_mock.well_known()['authorization_endpoint'] + authorization_url = kc_oidc_mock.well_known()["authorization_endpoint"] query_dict = QueryDict(parsed_url.query) # check redirect url is valid - assert urljoin(response['location'], parsed_url.path) == authorization_url - assert 'client_id' in query_dict - assert query_dict['client_id'] == OIDC_SWH_WEB_CLIENT_ID - assert 'response_type' in query_dict - assert query_dict['response_type'] == 'code' - assert 'redirect_uri' in query_dict - assert query_dict['redirect_uri'] == reverse('oidc-login-complete', - request=request) - assert 'code_challenge_method' in query_dict - assert query_dict['code_challenge_method'] == 'S256' - assert 'scope' in query_dict - assert query_dict['scope'] == 'openid' - assert 'state' in query_dict - assert 'code_challenge' in query_dict + assert urljoin(response["location"], parsed_url.path) == authorization_url + assert "client_id" in query_dict + assert query_dict["client_id"] == OIDC_SWH_WEB_CLIENT_ID + assert "response_type" in query_dict + assert query_dict["response_type"] == "code" + assert "redirect_uri" in query_dict + assert query_dict["redirect_uri"] == reverse("oidc-login-complete", request=request) + assert "code_challenge_method" in query_dict + assert query_dict["code_challenge_method"] == "S256" + assert "scope" in query_dict + assert query_dict["scope"] == "openid" + assert "state" in query_dict + assert "code_challenge" in query_dict # check a login_data has been registered in user session - assert 'login_data' in request.session - login_data = request.session['login_data'] - assert 'code_verifier' in login_data - assert 'state' in login_data - assert 'redirect_uri' in login_data - assert login_data['redirect_uri'] == query_dict['redirect_uri'] + assert "login_data" in request.session + login_data = request.session["login_data"] + assert "code_verifier" in login_data + assert "state" in login_data + assert "redirect_uri" in login_data + assert login_data["redirect_uri"] == query_dict["redirect_uri"] # once a user has identified himself in Keycloak, he is # redirected to the 'oidc-login-complete' view to # login in Django. # generate authorization code / session state in the same # manner as Keycloak - code = f'{str(uuid.uuid4())}.{str(uuid.uuid4())}.{str(uuid.uuid4())}' + code = f"{str(uuid.uuid4())}.{str(uuid.uuid4())}.{str(uuid.uuid4())}" session_state = str(uuid.uuid4()) - login_complete_url = reverse('oidc-login-complete', - query_params={'code': code, - 'state': login_data['state'], - 'session_state': session_state}) + login_complete_url = reverse( + "oidc-login-complete", + query_params={ + "code": code, + "state": login_data["state"], + "session_state": session_state, + }, + ) # login process finalization response = client.get(login_complete_url) request = response.wsgi_request # should redirect to root url by default assert response.status_code == 302 - assert response['location'] == request.build_absolute_uri('/') + assert response["location"] == request.build_absolute_uri("/") # user should be authenticated assert isinstance(request.user, OIDCUser) # check remote user has not been saved to Django database with pytest.raises(User.DoesNotExist): User.objects.get(username=request.user.username) @pytest.mark.django_db def test_oidc_logout_view_success(client, mocker): """ Simulate a successful logout operation with OpenID Connect. """ # mock Keycloak client kc_oidc_mock = mock_keycloak(mocker) # login our test user - client.login(code='', code_verifier='', redirect_uri='') + client.login(code="", code_verifier="", redirect_uri="") kc_oidc_mock.authorization_code.assert_called() # user initiates logout - oidc_logout_url = reverse('oidc-logout') + oidc_logout_url = reverse("oidc-logout") response = client.get(oidc_logout_url) request = response.wsgi_request # should redirect to logout page assert response.status_code == 302 - logout_url = reverse('logout', query_params={'remote_user': 1}) - assert response['location'] == request.build_absolute_uri(logout_url) + logout_url = reverse("logout", query_params={"remote_user": 1}) + assert response["location"] == request.build_absolute_uri(logout_url) # should have been logged out in Keycloak - kc_oidc_mock.logout.assert_called_with( - sample_data.oidc_profile['refresh_token']) + kc_oidc_mock.logout.assert_called_with(sample_data.oidc_profile["refresh_token"]) # check effective logout in Django assert isinstance(request.user, AnonymousUser) @pytest.mark.django_db def test_oidc_login_view_failure(client, mocker): """ Simulate a failed authentication with OpenID Connect. """ # mock Keycloak client mock_keycloak(mocker, auth_success=False) # user initiates login process - login_url = reverse('oidc-login') + login_url = reverse("oidc-login") response = client.get(login_url) request = response.wsgi_request # should render an error page assert response.status_code == 500 assert_template_used(response, "error.html") # no users should be logged in assert isinstance(request.user, AnonymousUser) # Simulate possible errors with OpenID Connect in the login complete view. + def test_oidc_login_complete_view_no_login_data(client, mocker): # user initiates login process - login_url = reverse('oidc-login-complete') + login_url = reverse("oidc-login-complete") response = client.get(login_url) # should render an error page assert_template_used(response, "error.html") - assert_contains(response, 'Login process has not been initialized.', - status_code=500) + assert_contains( + response, "Login process has not been initialized.", status_code=500 + ) def test_oidc_login_complete_view_missing_parameters(client, mocker): # simulate login process has been initialized session = client.session - session['login_data'] = { - 'code_verifier': '', - 'state': str(uuid.uuid4()), - 'redirect_uri': '', - 'next_path': '', - 'prompt': '', + session["login_data"] = { + "code_verifier": "", + "state": str(uuid.uuid4()), + "redirect_uri": "", + "next_path": "", + "prompt": "", } session.save() # user initiates login process - login_url = reverse('oidc-login-complete') + login_url = reverse("oidc-login-complete") response = client.get(login_url) request = response.wsgi_request # should render an error page assert_template_used(response, "error.html") - assert_contains(response, 'Missing query parameters for authentication.', - status_code=400) + assert_contains( + response, "Missing query parameters for authentication.", status_code=400 + ) # no user should be logged in assert isinstance(request.user, AnonymousUser) def test_oidc_login_complete_wrong_csrf_token(client, mocker): # mock Keycloak client mock_keycloak(mocker) # simulate login process has been initialized session = client.session - session['login_data'] = { - 'code_verifier': '', - 'state': str(uuid.uuid4()), - 'redirect_uri': '', - 'next_path': '', - 'prompt': '', + session["login_data"] = { + "code_verifier": "", + "state": str(uuid.uuid4()), + "redirect_uri": "", + "next_path": "", + "prompt": "", } session.save() # user initiates login process - login_url = reverse('oidc-login-complete', - query_params={'code': 'some-code', - 'state': 'some-state'}) + login_url = reverse( + "oidc-login-complete", query_params={"code": "some-code", "state": "some-state"} + ) response = client.get(login_url) request = response.wsgi_request # should render an error page assert_template_used(response, "error.html") - assert_contains(response, 'Wrong CSRF token, aborting login process.', - status_code=400) + assert_contains( + response, "Wrong CSRF token, aborting login process.", status_code=400 + ) # no user should be logged in assert isinstance(request.user, AnonymousUser) @pytest.mark.django_db def test_oidc_login_complete_wrong_code_verifier(client, mocker): # mock Keycloak client mock_keycloak(mocker, auth_success=False) # simulate login process has been initialized session = client.session - session['login_data'] = { - 'code_verifier': '', - 'state': str(uuid.uuid4()), - 'redirect_uri': '', - 'next_path': '', - 'prompt': '', + session["login_data"] = { + "code_verifier": "", + "state": str(uuid.uuid4()), + "redirect_uri": "", + "next_path": "", + "prompt": "", } session.save() # check authentication error is reported - login_url = reverse('oidc-login-complete', - query_params={'code': 'some-code', - 'state': session['login_data']['state']}) + login_url = reverse( + "oidc-login-complete", + query_params={"code": "some-code", "state": session["login_data"]["state"]}, + ) response = client.get(login_url) request = response.wsgi_request # should render an error page assert_template_used(response, "error.html") - assert_contains(response, 'User authentication failed.', - status_code=500) + assert_contains(response, "User authentication failed.", status_code=500) # no user should be logged in assert isinstance(request.user, AnonymousUser) @pytest.mark.django_db def test_oidc_logout_view_failure(client, mocker): """ Simulate a failed logout operation with OpenID Connect. """ # mock Keycloak client kc_oidc_mock = mock_keycloak(mocker) # login our test user - client.login(code='', code_verifier='', redirect_uri='') + client.login(code="", code_verifier="", redirect_uri="") - err_msg = 'Authentication server error' + err_msg = "Authentication server error" kc_oidc_mock.logout.side_effect = Exception(err_msg) # user initiates logout process - logout_url = reverse('oidc-logout') + logout_url = reverse("oidc-logout") response = client.get(logout_url) request = response.wsgi_request # should render an error page assert_template_used(response, "error.html") assert_contains(response, err_msg, status_code=500) # user should be logged out from Django anyway assert isinstance(request.user, AnonymousUser) @pytest.mark.django_db def test_oidc_silent_refresh_failure(client, mocker): # mock Keycloak client mock_keycloak(mocker) - next_path = reverse('swh-web-homepage') + next_path = reverse("swh-web-homepage") # silent session refresh initialization - login_url = reverse('oidc-login', query_params={'next_path': next_path, - 'prompt': 'none'}) + login_url = reverse( + "oidc-login", query_params={"next_path": next_path, "prompt": "none"} + ) response = client.get(login_url) request = response.wsgi_request - login_data = request.session['login_data'] + login_data = request.session["login_data"] # check prompt value has been registered in user session - assert 'prompt' in login_data - assert login_data['prompt'] == 'none' + assert "prompt" in login_data + assert login_data["prompt"] == "none" # simulate a failed silent session refresh session_state = str(uuid.uuid4()) - login_complete_url = reverse('oidc-login-complete', - query_params={'error': 'login_required', - 'state': login_data['state'], - 'session_state': session_state}) + login_complete_url = reverse( + "oidc-login-complete", + query_params={ + "error": "login_required", + "state": login_data["state"], + "session_state": session_state, + }, + ) # login process finalization response = client.get(login_complete_url) request = response.wsgi_request # should redirect to logout page assert response.status_code == 302 - logout_url = reverse('logout', query_params={'next_path': next_path, - 'remote_user': 1}) - assert response['location'] == logout_url + logout_url = reverse( + "logout", query_params={"next_path": next_path, "remote_user": 1} + ) + assert response["location"] == logout_url diff --git a/swh/web/tests/browse/test_utils.py b/swh/web/tests/browse/test_utils.py index d1c0c066..587497a9 100644 --- a/swh/web/tests/browse/test_utils.py +++ b/swh/web/tests/browse/test_utils.py @@ -1,128 +1,131 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import given from swh.web.browse import utils from swh.web.common.utils import reverse, format_utc_iso_date from swh.web.tests.strategies import origin_with_multiple_visits def test_get_mimetype_and_encoding_for_content(): - text = b'Hello world!' - assert (utils.get_mimetype_and_encoding_for_content(text) == - ('text/plain', 'us-ascii')) + text = b"Hello world!" + assert utils.get_mimetype_and_encoding_for_content(text) == ( + "text/plain", + "us-ascii", + ) @given(origin_with_multiple_visits()) def test_get_origin_visit_snapshot_simple(archive_data, origin): - visits = archive_data.origin_visit_get(origin['url']) + visits = archive_data.origin_visit_get(origin["url"]) for visit in visits: - snapshot = archive_data.snapshot_get(visit['snapshot']) + snapshot = archive_data.snapshot_get(visit["snapshot"]) branches = [] releases = [] def _process_branch_data(branch, branch_data): - if branch_data['target_type'] == 'revision': - rev_data = archive_data.revision_get(branch_data['target']) - branches.append({ - 'name': branch, - 'revision': branch_data['target'], - 'directory': rev_data['directory'], - 'date': format_utc_iso_date(rev_data['date']), - 'message': rev_data['message'] - }) - elif branch_data['target_type'] == 'release': - rel_data = archive_data.release_get(branch_data['target']) - rev_data = archive_data.revision_get(rel_data['target']) - releases.append({ - 'name': rel_data['name'], - 'branch_name': branch, - 'date': format_utc_iso_date(rel_data['date']), - 'id': rel_data['id'], - 'message': rel_data['message'], - 'target_type': rel_data['target_type'], - 'target': rel_data['target'], - 'directory': rev_data['directory'] - }) - - for branch in sorted(snapshot['branches'].keys()): - branch_data = snapshot['branches'][branch] - if branch_data['target_type'] == 'alias': - target_data = snapshot['branches'][branch_data['target']] + if branch_data["target_type"] == "revision": + rev_data = archive_data.revision_get(branch_data["target"]) + branches.append( + { + "name": branch, + "revision": branch_data["target"], + "directory": rev_data["directory"], + "date": format_utc_iso_date(rev_data["date"]), + "message": rev_data["message"], + } + ) + elif branch_data["target_type"] == "release": + rel_data = archive_data.release_get(branch_data["target"]) + rev_data = archive_data.revision_get(rel_data["target"]) + releases.append( + { + "name": rel_data["name"], + "branch_name": branch, + "date": format_utc_iso_date(rel_data["date"]), + "id": rel_data["id"], + "message": rel_data["message"], + "target_type": rel_data["target_type"], + "target": rel_data["target"], + "directory": rev_data["directory"], + } + ) + + for branch in sorted(snapshot["branches"].keys()): + branch_data = snapshot["branches"][branch] + if branch_data["target_type"] == "alias": + target_data = snapshot["branches"][branch_data["target"]] _process_branch_data(branch, target_data) else: _process_branch_data(branch, branch_data) - assert branches and releases, 'Incomplete test data.' + assert branches and releases, "Incomplete test data." origin_visit_branches = utils.get_origin_visit_snapshot( - origin, visit_id=visit['visit']) + origin, visit_id=visit["visit"] + ) assert origin_visit_branches == (branches, releases) def test_gen_link(): - assert (utils.gen_link('https://www.softwareheritage.org/', 'swh') == - 'swh') + assert ( + utils.gen_link("https://www.softwareheritage.org/", "swh") + == 'swh' + ) def test_gen_revision_link(): - revision_id = '28a0bc4120d38a394499382ba21d6965a67a3703' - revision_url = reverse('browse-revision', - url_args={'sha1_git': revision_id}) + revision_id = "28a0bc4120d38a394499382ba21d6965a67a3703" + revision_url = reverse("browse-revision", url_args={"sha1_git": revision_id}) - assert (utils.gen_revision_link(revision_id, link_text=None, - link_attrs=None) == - '%s' % (revision_url, revision_id)) - assert (utils.gen_revision_link(revision_id, shorten_id=True, - link_attrs=None) == - '%s' % (revision_url, revision_id[:7])) + assert utils.gen_revision_link( + revision_id, link_text=None, link_attrs=None + ) == '%s' % (revision_url, revision_id) + assert utils.gen_revision_link( + revision_id, shorten_id=True, link_attrs=None + ) == '%s' % (revision_url, revision_id[:7]) def test_gen_person_mail_link(): person_full = { - 'name': 'John Doe', - 'email': 'john.doe@swh.org', - 'fullname': 'John Doe ' + "name": "John Doe", + "email": "john.doe@swh.org", + "fullname": "John Doe ", } - assert (utils.gen_person_mail_link(person_full) == - '%s' % (person_full['email'], - person_full['name'])) + assert utils.gen_person_mail_link(person_full) == '%s' % ( + person_full["email"], + person_full["name"], + ) - link_text = 'Mail' - assert (utils.gen_person_mail_link(person_full, link_text=link_text) == - '%s' % (person_full['email'], - link_text)) + link_text = "Mail" + assert utils.gen_person_mail_link( + person_full, link_text=link_text + ) == '%s' % (person_full["email"], link_text) - person_partial_email = { - 'name': None, - 'email': None, - 'fullname': 'john.doe@swh.org' - } + person_partial_email = {"name": None, "email": None, "fullname": "john.doe@swh.org"} - assert (utils.gen_person_mail_link(person_partial_email) == - '%s' % (person_partial_email['fullname'], - person_partial_email['fullname'])) + assert utils.gen_person_mail_link( + person_partial_email + ) == '%s' % ( + person_partial_email["fullname"], + person_partial_email["fullname"], + ) person_partial = { - 'name': None, - 'email': None, - 'fullname': 'John Doe ' + "name": None, + "email": None, + "fullname": "John Doe ", } - assert (utils.gen_person_mail_link(person_partial) == - person_partial['fullname']) + assert utils.gen_person_mail_link(person_partial) == person_partial["fullname"] - person_none = { - 'name': None, - 'email': None, - 'fullname': None - } + person_none = {"name": None, "email": None, "fullname": None} - assert utils.gen_person_mail_link(person_none) == 'None' + assert utils.gen_person_mail_link(person_none) == "None" diff --git a/swh/web/tests/browse/views/test_content.py b/swh/web/tests/browse/views/test_content.py index 267358d1..b96130dc 100644 --- a/swh/web/tests/browse/views/test_content.py +++ b/swh/web/tests/browse/views/test_content.py @@ -1,386 +1,388 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.utils.html import escape from hypothesis import given from swh.web.browse.utils import ( - get_mimetype_and_encoding_for_content, prepare_content_for_display, - _re_encode_content + get_mimetype_and_encoding_for_content, + prepare_content_for_display, + _re_encode_content, ) from swh.web.common.exc import NotFoundExc from swh.web.common.identifiers import get_swh_persistent_id from swh.web.common.utils import gen_path_info, reverse from swh.web.tests.django_asserts import ( - assert_contains, assert_not_contains, assert_template_used + assert_contains, + assert_not_contains, + assert_template_used, ) from swh.web.tests.strategies import ( - content, content_text_non_utf8, content_text_no_highlight, - content_image_type, content_text, invalid_sha1, unknown_content, - content_utf8_detected_as_binary + content, + content_text_non_utf8, + content_text_no_highlight, + content_image_type, + content_text, + invalid_sha1, + unknown_content, + content_utf8_detected_as_binary, ) @given(content_text()) def test_content_view_text(client, archive_data, content): - sha1_git = content['sha1_git'] + sha1_git = content["sha1_git"] - url = reverse('browse-content', - url_args={'query_string': content['sha1']}, - query_params={'path': content['path']}) + url = reverse( + "browse-content", + url_args={"query_string": content["sha1"]}, + query_params={"path": content["path"]}, + ) - url_raw = reverse('browse-content-raw', - url_args={'query_string': content['sha1']}) + url_raw = reverse("browse-content-raw", url_args={"query_string": content["sha1"]}) resp = client.get(url) content_display = _process_content_for_display(archive_data, content) - mimetype = content_display['mimetype'] + mimetype = content_display["mimetype"] assert resp.status_code == 200 - assert_template_used(resp, 'browse/content.html') + assert_template_used(resp, "browse/content.html") - if mimetype.startswith('text/'): - assert_contains(resp, '' % - content_display['language']) - assert_contains(resp, escape(content_display['content_data'])) + if mimetype.startswith("text/"): + assert_contains(resp, '' % content_display["language"]) + assert_contains(resp, escape(content_display["content_data"])) assert_contains(resp, url_raw) - swh_cnt_id = get_swh_persistent_id('content', sha1_git) - swh_cnt_id_url = reverse('browse-swh-id', - url_args={'swh_id': swh_cnt_id}) + swh_cnt_id = get_swh_persistent_id("content", sha1_git) + swh_cnt_id_url = reverse("browse-swh-id", url_args={"swh_id": swh_cnt_id}) assert_contains(resp, swh_cnt_id) assert_contains(resp, swh_cnt_id_url) @given(content_text_no_highlight()) def test_content_view_text_no_highlight(client, archive_data, content): - sha1_git = content['sha1_git'] + sha1_git = content["sha1_git"] - url = reverse('browse-content', - url_args={'query_string': content['sha1']}) + url = reverse("browse-content", url_args={"query_string": content["sha1"]}) - url_raw = reverse('browse-content-raw', - url_args={'query_string': content['sha1']}) + url_raw = reverse("browse-content-raw", url_args={"query_string": content["sha1"]}) resp = client.get(url) content_display = _process_content_for_display(archive_data, content) assert resp.status_code == 200 - assert_template_used(resp, 'browse/content.html') + assert_template_used(resp, "browse/content.html") assert_contains(resp, '') - assert_contains(resp, escape(content_display['content_data'])) + assert_contains(resp, escape(content_display["content_data"])) assert_contains(resp, url_raw) - swh_cnt_id = get_swh_persistent_id('content', sha1_git) - swh_cnt_id_url = reverse('browse-swh-id', - url_args={'swh_id': swh_cnt_id}) + swh_cnt_id = get_swh_persistent_id("content", sha1_git) + swh_cnt_id_url = reverse("browse-swh-id", url_args={"swh_id": swh_cnt_id}) assert_contains(resp, swh_cnt_id) assert_contains(resp, swh_cnt_id_url) @given(content_text_non_utf8()) def test_content_view_no_utf8_text(client, archive_data, content): - sha1_git = content['sha1_git'] + sha1_git = content["sha1_git"] - url = reverse('browse-content', - url_args={'query_string': content['sha1']}) + url = reverse("browse-content", url_args={"query_string": content["sha1"]}) resp = client.get(url) content_display = _process_content_for_display(archive_data, content) assert resp.status_code == 200 - assert_template_used(resp, 'browse/content.html') - swh_cnt_id = get_swh_persistent_id('content', sha1_git) - swh_cnt_id_url = reverse('browse-swh-id', - url_args={'swh_id': swh_cnt_id}) + assert_template_used(resp, "browse/content.html") + swh_cnt_id = get_swh_persistent_id("content", sha1_git) + swh_cnt_id_url = reverse("browse-swh-id", url_args={"swh_id": swh_cnt_id}) assert_contains(resp, swh_cnt_id_url) - assert_contains(resp, escape(content_display['content_data'])) + assert_contains(resp, escape(content_display["content_data"])) @given(content_image_type()) def test_content_view_image(client, archive_data, content): - url = reverse('browse-content', - url_args={'query_string': content['sha1']}) + url = reverse("browse-content", url_args={"query_string": content["sha1"]}) - url_raw = reverse('browse-content-raw', - url_args={'query_string': content['sha1']}) + url_raw = reverse("browse-content-raw", url_args={"query_string": content["sha1"]}) resp = client.get(url) content_display = _process_content_for_display(archive_data, content) - mimetype = content_display['mimetype'] - content_data = content_display['content_data'] + mimetype = content_display["mimetype"] + content_data = content_display["content_data"] assert resp.status_code == 200 - assert_template_used(resp, 'browse/content.html') - assert_contains(resp, '' - % (mimetype, content_data)) + assert_template_used(resp, "browse/content.html") + assert_contains(resp, '' % (mimetype, content_data)) assert_contains(resp, url_raw) @given(content_text()) def test_content_view_text_with_path(client, archive_data, content): - path = content['path'] + path = content["path"] - url = reverse('browse-content', - url_args={'query_string': content['sha1']}, - query_params={'path': path}) + url = reverse( + "browse-content", + url_args={"query_string": content["sha1"]}, + query_params={"path": path}, + ) resp = client.get(url) assert resp.status_code == 200 - assert_template_used(resp, 'browse/content.html') + assert_template_used(resp, "browse/content.html") assert_contains(resp, '